{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 导包\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import seaborn as sns\n",
    "import jieba\n",
    "\n",
    "import collections\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('b', 4), ('d', 3), ('a', 2)]\n",
      "0.5312500000000001\n",
      "{'交通': 1, '车': 2, '巴': 3, '索道': 4, '飞机': 5, '住宿': 6, '酒店': 7, '客栈': 8, '旅馆': 9, '民宿': 10, '房间': 11, '宾馆': 12, '床': 13, '窗': 14, '食物': 15, '吃': 16, '味道': 17, '饭': 18, '菜': 19, '干粮': 20, '食品': 21, '零食': 22, '带水': 23, '水果': 24, '饮料': 25, '矿泉水': 26, '餐馆': 27, '饭店': 28, '景': 29, '山水': 30, '风光': 31, '环境': 32, '导游': 33, '司机': 34, '讲解': 35, '解说': 36, '工作人员': 37, '旺季': 38, '节假日': 39, '高峰': 40, '国庆': 41, '春节': 42, '黄金周': 43, '十一': 44, '拥挤': 45, '热闹': 46, '堵': 47, '吵闹': 48, '安静': 49, '轻松': 50, '宁静': 51, '清静': 52, '幽静': 53, '悠闲': 54, '平静': 55, '坑': 56, '骗': 57, '失望': 58, '忽悠': 59, '亲子': 60, '家庭': 61, '儿童': 62, '儿子': 63, '双人': 64, '微信': 65, '厕所': 66, '烧香': 67}\n",
      "deque([], maxlen=3)\n",
      "\n",
      "0\n",
      "1207\n",
      "['', '呼啦', '之一', '当', '；', '得', '不经意', '传', '吗', '不但', '喔唷', '五', '饱', '逢', '哗', '当庭', '每时每刻', '从不', '日见', '如期', '待', '然则', '从古至今', '串行', '起来', '可见', '经常', '没', '那个', '上去', '别说', '紧接着', '当然', '嘎嘎', '啥', '不时', '为着', '向着', '趁早', '敞开儿', '对', '忽然', '倘使', '互相', '匆匆', '１', '成年累月', '切勿', '又', '那', '另一方面', '如果', '就是', '跟', '依照', '看起来', '各式', '临到', '1', '恐怕', '迟早', '基本', '一', '并没有', '大家', '猛然', '通过', '除去', '#', '从而', '除了', '要不然', '６', '不但...而且', '常常', '当头', '另一个', '并肩', '不起', '>', '权时', '话说', '多亏', '乃至', '哉', '简言之', '借此', '几乎', '八', '或多或少', '可能', '切切', '每当', '趁', '吓', '!', '倍感', '上下', '甚么', '它']\n"
     ]
    }
   ],
   "source": [
    "# 函数定义\n",
    "from collections import Counter\n",
    "from snownlp import SnowNLP\n",
    "\n",
    "def merge(path, fileName):\n",
    "    trainName = '../models/__models__/{}_{}.csv'.format(path, fileName)\n",
    "    testName = trainName.replace('train', 'test')\n",
    "    train = pd.read_csv(trainName)\n",
    "    test = pd.read_csv(testName)\n",
    "    data = pd.concat([train, test])\n",
    "    return data\n",
    "\n",
    "def read_map(filename):\n",
    "    with open(Configure.root_data_path + filename, encoding='utf-8') as f:\n",
    "        dictionary = dict()\n",
    "        for line in f.readlines():\n",
    "            line = line.strip().split(' ')\n",
    "            dictionary[line[0]] = line[1]\n",
    "        return dictionary\n",
    "\n",
    "def read_file_word2dict(filename):\n",
    "    with open(filename, 'r') as f:\n",
    "        dictionary = dict()\n",
    "        for line in f.readlines():\n",
    "            line = line.strip().split(' ')\n",
    "            for word in line:\n",
    "                if word not in dictionary:\n",
    "                    dictionary[word] = len(dictionary) + 1\n",
    "        return dictionary\n",
    "    \n",
    "def read_file_word2set(filename, encoding = 'utf-8'):\n",
    "    with open(filename, 'r', encoding = encoding) as f:\n",
    "        dictionary = set()\n",
    "        for line in f.readlines():\n",
    "            line = line.strip().split(' ')\n",
    "            for word in line:\n",
    "                dictionary.add(word)\n",
    "        return dictionary\n",
    "\n",
    "def get_neg_words(data_df, stop_set = set(), feature = 'words', label = 'Score'):\n",
    "    data_df_neg = data_df[data_df[label] == 1]\n",
    "    return get_words(data_df_neg, stop_set = stop_set, feature = feature)\n",
    "\n",
    "def get_keywords(data_df, stop_set = set(), feature = 'key_word', label = -1):\n",
    "    dictionary = []\n",
    "    score = data_df['Score'].values\n",
    "    for i, word in enumerate(data_df[feature].values):\n",
    "        word = str(word)\n",
    "        if len(stop_set) == 0:\n",
    "            if word != 'nan' and score[i] == label:\n",
    "                dictionary.append(word)\n",
    "        else:\n",
    "            if word != 'nan' and score[i] == label and word not in stop_set:\n",
    "                dictionary.append(word)\n",
    "    return dictionary\n",
    "\n",
    "def get_words(data_df, stop_set = set(), feature = 'words'):\n",
    "    data_df = data_df[feature].values\n",
    "    dictionary = []\n",
    "    for sent in data_df:\n",
    "        sent = sent[1:-1]\n",
    "        sent = sent.split(';')\n",
    "        for word in sent:\n",
    "            if len(stop_set) == 0:\n",
    "                if word != '':\n",
    "                    dictionary.append(word)\n",
    "            else:\n",
    "                if word != '' and word not in stop_set:\n",
    "                    dictionary.append(word)\n",
    "    return dictionary\n",
    "\n",
    "def build_dataset(words, vocabulary_size = 5000):\n",
    "    from collections import Counter\n",
    "    count = [['UNK', -1]]\n",
    "    count.extend(Counter(words).most_common(vocabulary_size - 1))\n",
    "    dictionary = dict()\n",
    "    for word, _ in count:\n",
    "        dictionary[word] = len(dictionary)\n",
    "    data = list()\n",
    "    unk_count = 0\n",
    "    for word in words:\n",
    "        if word in dictionary:\n",
    "            index = dictionary[word]\n",
    "        else:\n",
    "            index = 0\n",
    "            unk_count += 1\n",
    "        data.append(index)\n",
    "    count[0][1] = unk_count\n",
    "    reverse_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "    return data, count, dictionary, reverse_dictionary\n",
    "\n",
    "\n",
    "def senti_snownlp(dicuss):\n",
    "    if dicuss == '': return 0.5\n",
    "    return SnowNLP(dicuss).sentiments\n",
    "\n",
    "def clean_str(x, stop_word = '，'):\n",
    "    words = list(jieba.cut(x))\n",
    "    strbuild = '['\n",
    "    for word in words:\n",
    "        if word != stop_word:\n",
    "            strbuild += word + ';'\n",
    "    if ';' in strbuild: strbuild = strbuild[:-1]\n",
    "    strbuild += ']'\n",
    "    return strbuild    \n",
    "\n",
    "def clean_str(words, stop_list):\n",
    "    words = words[1:-1]\n",
    "    words = words.split(';')\n",
    "    strbuild = '['\n",
    "    for word in words:\n",
    "        if word not in stop_list:\n",
    "            strbuild += word + ';'\n",
    "    if ';' in strbuild: strbuild = strbuild[:-1]\n",
    "    strbuild += ']'\n",
    "    return strbuild    \n",
    "\n",
    "def getdictionary(topK = 10, stop_set = set(), label = 5, maxLen = 5000):\n",
    "    data_keyword = pd.read_csv(model_path + 'data_keyword.csv')\n",
    "    data_score = data.copy()\n",
    "    data_score = pd.merge(data_keyword, data_score[['Id', 'Score']], on = 'Id', how = 'left')\n",
    "    all_words = []\n",
    "    for i in range(topK):\n",
    "        words = get_keywords(data_score, stop_set = stop_set, feature = 'key_word_' + str(i), label = label)\n",
    "        all_words.extend(words)\n",
    "    word_num, count, dictionary, reverse_dictionary = build_dataset(all_words, vocabulary_size=min(maxLen, len(all_words)))\n",
    "    print(count[0:100])\n",
    "    return dictionary\n",
    "\n",
    "# test\n",
    "from collections import Counter\n",
    "print(Counter(['a', 'a', 'v', 'b', 'b', 'b', 'c', 'd', 'd', 'd', 'b']).most_common(3))\n",
    "\n",
    "print(senti_snownlp('你好'))\n",
    "\n",
    "significant_dictionary = read_file_word2dict('../input/significance.txt')\n",
    "print(significant_dictionary)\n",
    "\n",
    "import collections\n",
    "buffer = collections.deque(maxlen=3)\n",
    "print(buffer)\n",
    "print(''.join(list(buffer)))\n",
    "print(len(buffer))\n",
    "\n",
    "# 停用词 读取\n",
    "stop_list = read_file_word2set('../input/stop_word.txt')\n",
    "print(len(stop_list))\n",
    "print(list(stop_list)[0:100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEKCAYAAADaa8itAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAAFaRJREFUeJzt3X/wXXV95/HnSwKVVSlQvrA0gY27TVVqq0IKcdmxW3FCsFaYjuziVkktO+k46OBsp13cnSmt1lk7u60Vf+0yEk1ct8hqXaKLxixCHStCgiAI6JKlVr4TloQGEOuWDvS9f9xPmmu4Sb4kn5vz/ZLnY+bOPed9Pud83+f+kVfOj3tuqgpJknp4ztANSJKePQwVSVI3hookqRtDRZLUjaEiSerGUJEkdWOoSJK6MVQkSd0YKpKkbhYN3cChdsIJJ9TSpUuHbkOSFozbbrvt4aqamcvYwy5Uli5dypYtW4ZuQ5IWjCR/Odexnv6SJHVjqEiSujFUJEndGCqSpG4MFUlSN1MNlSTHJvl0km8nuTfJK5Mcn2RTkvva+3FtbJJcmWRrkjuTnD62ndVt/H1JVo/Vz0hyV1vnyiSZ5v5IkvZt2kcq7we+WFUvBl4G3AtcDtxQVcuAG9o8wHnAsvZaA3wEIMnxwBXAWcCZwBW7gqiNWTO23qop748kaR+mFipJjgFeBVwNUFV/W1WPAucD69qwdcAFbfp8YH2NfB04NsnJwLnApqraWVWPAJuAVW3ZMVV1c41+E3n92LYkSQOY5pHKPwZ2AB9LcnuSjyZ5HnBSVT0I0N5PbOMXAw+MrT/bavuqz06oS5IGMs1v1C8CTgfeXlW3JHk/u091TTLpekgdQP3pG07WMDpNxqmnnrqvniVpvz74m58buoWpeNsf/vJBb2OaRyqzwGxV3dLmP80oZB5qp65o79vHxp8ytv4SYNt+6ksm1J+mqq6qquVVtXxmZk6Pr5EkHYCphUpV/V/ggSQvaqVzgHuADcCuO7hWA9e16Q3Axe0usBXAY+302EZgZZLj2gX6lcDGtuzxJCvaXV8Xj21LkjSAaT9Q8u3AJ5McBdwPvIVRkF2b5BLge8CFbez1wGuBrcAP21iqameSdwOb27h3VdXONv1W4OPA0cAX2kuSNJCphkpV3QEsn7DonAljC7h0L9tZC6ydUN8CvPQg25QkdeI36iVJ3RgqkqRuDBVJUjeGiiSpG0NFktSNoSJJ6sZQkSR1Y6hIkroxVCRJ3RgqkqRuDBVJUjeGiiSpG0NFktSNoSJJ6sZQkSR1Y6hIkroxVCRJ3RgqkqRuDBVJUjeGiiSpG0NFktSNoSJJ6sZQkSR1Y6hIkroxVCRJ3RgqkqRuphoqSb6b5K4kdyTZ0mrHJ9mU5L72flyrJ8mVSbYmuTPJ6WPbWd3G35dk9Vj9jLb9rW3dTHN/JEn7diiOVH6xql5eVcvb/OXADVW1DLihzQOcByxrrzXAR2AUQsAVwFnAmcAVu4KojVkztt6q6e+OJGlvhjj9dT6wrk2vAy4Yq6+vka8DxyY5GTgX2FRVO6vqEWATsKotO6aqbq6qAtaPbUuSNIBph0oBX0pyW5I1rXZSVT0I0N5PbPXFwANj68622r7qsxPqkqSBLJry9s+uqm1JTgQ2Jfn2PsZOuh5SB1B/+oZHgbYG4NRTT913x5KkAzbVI5Wq2tbetwOfZXRN5KF26or2vr0NnwVOGVt9CbBtP/UlE+qT+riqqpZX1fKZmZmD3S1J0l5MLVSSPC/JC3ZNAyuBbwEbgF13cK0GrmvTG4CL211gK4DH2umxjcDKJMe1C/QrgY1t2eNJVrS7vi4e25YkaQDTPP11EvDZdpfvIuC/VdUXk2wGrk1yCfA94MI2/nrgtcBW4IfAWwCqameSdwOb27h3VdXONv1W4OPA0cAX2kuSNJCphUpV3Q+8bEL9r4BzJtQLuHQv21oLrJ1Q3wK89KCblSR14TfqJUndGCqSpG4MFUlSN4aKJKkbQ0WS1I2hIknqxlCRJHVjqEiSujFUJEndGCqSpG4MFUlSN4aKJKkbQ0WS1I2hIknqxlCRJHVjqEiSujFUJEndGCqSpG4MFUlSN4aKJKkbQ0WS1I2hIknqxlCRJHVjqEiSujFUJEndTD1UkhyR5PYkn2/zL0xyS5L7knwqyVGt/mNtfmtbvnRsG+9s9e8kOXesvqrVtia5fNr7Iknat0NxpHIZcO/Y/B8A76uqZcAjwCWtfgnwSFX9FPC+No4kpwEXAT8DrAI+3ILqCOBDwHnAacAb21hJ0kCmGipJlgC/BHy0zQd4NfDpNmQdcEGbPr/N05af08afD1xTVU9U1V8AW4Ez22trVd1fVX8LXNPGSpIGMu0jlT8Gfhv4uzb/E8CjVfVkm58FFrfpxcADAG35Y23839f3WGdvdUnSQKYWKkleB2yvqtvGyxOG1n6WPdP6pF7WJNmSZMuOHTv20bUk6WBM80jlbOD1Sb7L6NTUqxkduRybZFEbswTY1qZngVMA2vIfB3aO1/dYZ2/1p6mqq6pqeVUtn5mZOfg9kyRNNLVQqap3VtWSqlrK6EL7l6vqV4EbgTe0YauB69r0hjZPW/7lqqpWv6jdHfZCYBlwK7AZWNbuJjuq/Y0N09ofSdL+Ldr/kO7+LXBNkt8HbgeubvWrgU8k2croCOUigKq6O8m1wD3Ak8ClVfUUQJK3ARuBI4C1VXX3Id0TSdKPOCShUlU3ATe16fsZ3bm155i/AS7cy/rvAd4zoX49cH3HViVJB8Fv1EuSujFUJEndGCqSpG4MFUlSN4aKJKkbQ0WS1I2hIknqxlCRJHVjqEiSujFUJEndGCqSpG4MFUlSN3MKlSQ3zKUmSTq87fMpxUmeC/wD4IQkx7H71xaPAX5yyr1JkhaY/T36/jeAdzAKkNvYHSrfBz40xb4kSQvQPkOlqt4PvD/J26vqA4eoJ0nSAjWnH+mqqg8k+afA0vF1qmr9lPqSJC1AcwqVJJ8A/glwB/BUKxdgqEiS/t5cf054OXBaVdU0m5EkLWxz/Z7Kt4B/OM1GJEkL31yPVE4A7klyK/DErmJVvX4qXUmSFqS5hsrvTrMJSdKzw1zv/vqzaTciSVr45nr31+OM7vYCOAo4EvjrqjpmWo1JkhaeuR6pvGB8PskFwJlT6UiStGAd0FOKq+p/AK/u3IskaYGb61OKf2Xs9YYk72X36bC9rfPcJLcm+WaSu5P8Xqu/MMktSe5L8qkkR7X6j7X5rW350rFtvbPVv5Pk3LH6qlbbmuTyA9h/SVJHcz1S+eWx17nA48D5+1nnCeDVVfUy4OXAqiQrgD8A3ldVy4BHgEva+EuAR6rqp4D3tXEkOQ24CPgZYBXw4SRHJDmC0UMtzwNOA97YxkqSBjLXaypveaYbbt++/0GbPbK9itFps3/V6usY3a78EUYh9but/mngg0nS6tdU1RPAXyTZyu7rOVur6n6AJNe0sfc8014lSX3M9fTXkiSfTbI9yUNJPpNkyRzWOyLJHcB2YBPwf4BHq+rJNmQWWNymFwMPALTljwE/MV7fY5291Sf1sSbJliRbduzYMZddliQdgLme/voYsIHR76osBj7XavtUVU9V1cuBJYyOLl4yaVh7z16WPdP6pD6uqqrlVbV8ZmZmf21Lkg7QXENlpqo+VlVPttfHgTn/61xVjwI3ASuAY5PsOu22BNjWpmeBUwDa8h8Hdo7X91hnb3VJ0kDmGioPJ3nTrgvkSd4E/NW+Vkgyk+TYNn008BrgXuBG4A1t2Grguja9oc3Tln+5XZfZAFzU7g57IbAMuBXYDCxrd5Mdxehi/oY57o8kaQrm+uyvXwc+yOiurAK+Buzv4v3JwLp2l9ZzgGur6vNJ7gGuSfL7wO3A1W381cAn2oX4nYxCgqq6O8m1jC7APwlcWlVPASR5G7AROAJYW1V3z3F/JD1Df/aqXxi6han4ha/4FKqe5hoq7wZWV9UjAEmOB/4To7CZqKruBF4xoX4/E76NX1V/A1y4l229B3jPhPr1wPVz2wVJ0rTN9fTXz+0KFICq2smEwJAkHd7mGirPSXLcrpl2pDLXoxxJ0mFirsHwh8DXknya0TWVf8GE01GSpMPbXL9Rvz7JFkbfhg/wK1XlN9clST9izqewWogYJJKkvTqgR99LkjSJoSJJ6sZQkSR1Y6hIkroxVCRJ3RgqkqRuDBVJUjeGiiSpG0NFktSNoSJJ6sZQkSR1Y6hIkroxVCRJ3RgqkqRuDBVJUjeGiiSpG0NFktSNoSJJ6sZQkSR1Y6hIkrqZWqgkOSXJjUnuTXJ3ksta/fgkm5Lc196Pa/UkuTLJ1iR3Jjl9bFur2/j7kqweq5+R5K62zpVJMq39kSTt3zSPVJ4EfrOqXgKsAC5NchpwOXBDVS0DbmjzAOcBy9prDfARGIUQcAVwFnAmcMWuIGpj1oytt2qK+yNJ2o+phUpVPVhV32jTjwP3AouB84F1bdg64II2fT6wvka+Dhyb5GTgXGBTVe2sqkeATcCqtuyYqrq5qgpYP7YtSdIADsk1lSRLgVcAtwAnVdWDMAoe4MQ2bDHwwNhqs622r/rshLokaSBTD5Ukzwc+A7yjqr6/r6ETanUA9Uk9rEmyJcmWHTt27K9lSdIBmmqoJDmSUaB8sqr+tJUfaqeuaO/bW30WOGVs9SXAtv3Ul0yoP01VXVVVy6tq+czMzMHtlCRpr6Z591eAq4F7q+qPxhZtAHbdwbUauG6sfnG7C2wF8Fg7PbYRWJnkuHaBfiWwsS17PMmK9rcuHtuWJGkAi6a47bOBNwN3Jbmj1f4d8F7g2iSXAN8DLmzLrgdeC2wFfgi8BaCqdiZ5N7C5jXtXVe1s028FPg4cDXyhvSRJA5laqFTVV5l83QPgnAnjC7h0L9taC6ydUN8CvPQg2pQkdeQ36iVJ3RgqkqRuDBVJUjeGiiSpG0NFktSNoSJJ6sZQkSR1Y6hIkroxVCRJ3RgqkqRuDBVJUjeGiiSpG0NFktSNoSJJ6sZQkSR1Y6hIkroxVCRJ3RgqkqRuDBVJUjeGiiSpG0NFktSNoSJJ6sZQkSR1Y6hIkroxVCRJ3RgqkqRuphYqSdYm2Z7kW2O145NsSnJfez+u1ZPkyiRbk9yZ5PSxdVa38fclWT1WPyPJXW2dK5NkWvsiSZqbaR6pfBxYtUftcuCGqloG3NDmAc4DlrXXGuAjMAoh4ArgLOBM4IpdQdTGrBlbb8+/JUk6xKYWKlX1FWDnHuXzgXVteh1wwVh9fY18HTg2ycnAucCmqtpZVY8Am4BVbdkxVXVzVRWwfmxbkqSBHOprKidV1YMA7f3EVl8MPDA2brbV9lWfnVCfKMmaJFuSbNmxY8dB74QkabL5cqF+0vWQOoD6RFV1VVUtr6rlMzMzB9iiJGl/Fh3iv/dQkpOr6sF2Cmt7q88Cp4yNWwJsa/V/vkf9plZfMmG81NXZHzh76Bam4s/f/udDt6BnqUN9pLIB2HUH12rgurH6xe0usBXAY+302EZgZZLj2gX6lcDGtuzxJCvaXV8Xj21LkjSQqR2pJPkTRkcZJySZZXQX13uBa5NcAnwPuLANvx54LbAV+CHwFoCq2pnk3cDmNu5dVbXr4v9bGd1hdjTwhfaSJA1oaqFSVW/cy6JzJowt4NK9bGctsHZCfQvw0oPpUZLU13y5UC9JehYwVCRJ3RgqkqRuDBVJUjeGiiSpG0NFktSNoSJJ6sZQkSR1Y6hIkroxVCRJ3RgqkqRuDBVJUjeGiiSpG0NFktSNoSJJ6sZQkSR1Y6hIkroxVCRJ3RgqkqRuDBVJUjeGiiSpG0NFktSNoSJJ6sZQkSR1Y6hIkrpZNHQDByvJKuD9wBHAR6vqvQO3tOB9710/O3QLU3Hq79w1dAvSs96CPlJJcgTwIeA84DTgjUlOG7YrSTp8LfQjlTOBrVV1P0CSa4DzgXue6YbO+K31nVubH277jxcP3YKkw8iCPlIBFgMPjM3PtpokaQCpqqF7OGBJLgTOrap/3ebfDJxZVW/fY9waYE2bfRHwnUPa6NOdADw8cA/zhZ/Fbn4Wu/lZ7DYfPot/VFUzcxm40E9/zQKnjM0vAbbtOaiqrgKuOlRN7U+SLVW1fOg+5gM/i938LHbzs9htoX0WC/3012ZgWZIXJjkKuAjYMHBPknTYWtBHKlX1ZJK3ARsZ3VK8tqruHrgtSTpsLehQAaiq64Hrh+7jGZo3p+LmAT+L3fwsdvOz2G1BfRYL+kK9JGl+WejXVCRJ84ihcgglWZtke5JvDd3L0JKckuTGJPcmuTvJZUP3NJQkz01ya5Jvts/i94buaUhJjkhye5LPD93L0JJ8N8ldSe5IsmXofubC01+HUJJXAT8A1lfVS4fuZ0hJTgZOrqpvJHkBcBtwQVU946chLHRJAjyvqn6Q5Ejgq8BlVfX1gVsbRJJ/AywHjqmq1w3dz5CSfBdYXlVDf09lzjxSOYSq6ivAzqH7mA+q6sGq+kabfhy4l8P0aQg18oM2e2R7HZb/20uyBPgl4KND96IDY6hocEmWAq8Abhm2k+G0Uz53ANuBTVV1uH4Wfwz8NvB3QzcyTxTwpSS3tSeDzHuGigaV5PnAZ4B3VNX3h+5nKFX1VFW9nNFTIc5MctidHk3yOmB7Vd02dC/zyNlVdTqjJ7Ff2k6hz2uGigbTrh98BvhkVf3p0P3MB1X1KHATsGrgVoZwNvD6dh3hGuDVSf7rsC0Nq6q2tfftwGcZPZl9XjNUNIh2cfpq4N6q+qOh+xlSkpkkx7bpo4HXAN8etqtDr6reWVVLqmopo0cufbmq3jRwW4NJ8rx2EwtJngesBOb9naOGyiGU5E+Am4EXJZlNcsnQPQ3obODNjP43ekd7vXbopgZyMnBjkjsZPc9uU1Ud9rfTipOAryb5JnAr8D+r6osD97Rf3lIsSerGIxVJUjeGiiSpG0NFktSNoSJJ6sZQkSR1Y6hIU5Lk37enDt/Zbpk+a+iepGlb8L/8KM1HSV4JvA44vaqeSHICcNRBbG9RVT3ZrUFpSjxSkabjZODhqnoCoKoerqptSX4+ydfab6fcmuQF7fdUPtZ+N+P2JL8IkOTXkvz3JJ8DvtRqv5Vkczv6Oax/d0Xzk0cq0nR8CfidJP8b+F/Apxg9TeFTwL+sqs1JjgH+H3AZQFX9bJIXM3oq7U+37bwS+Lmq2plkJbCM0fOfAmxI8qr2kwrSvOCRijQF7fdRzgDWADsYhclvAA9W1eY25vvtlNY/Az7Rat8G/hLYFSqbqmrXb/CsbK/bgW8AL2YUMtK84ZGKNCVV9RSjJw7flOQu4FIm//hW9rGZv95j3H+oqv/SrUmpM49UpClI8qIk40cRL2f065Y/meTn25gXJFkEfAX41Vb7aeBU4DsTNrsR+PX2GzQkWZzkxCnuhvSMeaQiTcfzgQ+0R9o/CWxldCrsY61+NKPrKa8BPgz853Y08yTwa+2OsR/ZYFV9KclLgJvbsh8Ab2L0a5HSvOBTiiVJ3Xj6S5LUjaEiSerGUJEkdWOoSJK6MVQkSd0YKpKkbgwVSVI3hookqZv/D7JtNDRwVuv2AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5 4 3 2 1]\n",
      "(100000, 3) (30000, 2)\n"
     ]
    }
   ],
   "source": [
    "# 加载数据集\n",
    "model_path = '../input/'\n",
    "testFileExist = True\n",
    "\n",
    "trainFile = model_path + 'train_first.csv'\n",
    "testFile = model_path + 'predict_first.csv'\n",
    "\n",
    "train = pd.read_csv(trainFile)\n",
    "test = pd.read_csv(testFile)\n",
    "\n",
    "# 数据清洗\n",
    "# train.drop_duplicates(subset = 'Discuss', keep = 'first', inplace = True)\n",
    "\n",
    "sns.countplot(x = 'Score',data = train)\n",
    "plt.show()\n",
    "\n",
    "print(train['Score'].unique())  # 采取多分类？\n",
    "print(train.shape, test.shape)\n",
    "\n",
    "\n",
    "test['Score'] = -1\n",
    "data = pd.concat([train, test])\n",
    "\n",
    "# model_path = '../input/subset12/'\n",
    "# data = pd.read_csv(model_path + 'train_score12.csv')\n",
    "# testFileExist = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "senti_snownlp file exists...\n",
      "data_jieba file exists...\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>words_jieba</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>201e8bf2-77a2-3a98-9fcf-4ce03914e712</td>\n",
       "      <td>[好大;的;一个;游乐;公园;已经;去;了;2;次;但;感觉;还;没有;玩够;似的;！;会;...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>bc578e6d-123e-3833-9a46-d0b4d7dccd0c</td>\n",
       "      <td>[景色;秀丽]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5e03c67a-8103-3b61-9052-82e269cf1c48</td>\n",
       "      <td>[辉煌;与;屈辱;也;都;成为;了;过去]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>68988e16-af63-34a2-a9aa-8e594d492a3a</td>\n",
       "      <td>[东方明珠;广播;电视塔;是;上海;的;标志性;文化景观;之一;位于;浦东新区;陆家嘴;塔高...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>e1813faf-2268-32b3-9bcf-41cb845e6728</td>\n",
       "      <td>[特色;的;建筑;各式;的;小店;好吃;的;小吃;还有;沙滩;和;朋友;和;家人;去;游玩;...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Id  \\\n",
       "0  201e8bf2-77a2-3a98-9fcf-4ce03914e712   \n",
       "1  bc578e6d-123e-3833-9a46-d0b4d7dccd0c   \n",
       "2  5e03c67a-8103-3b61-9052-82e269cf1c48   \n",
       "3  68988e16-af63-34a2-a9aa-8e594d492a3a   \n",
       "4  e1813faf-2268-32b3-9bcf-41cb845e6728   \n",
       "\n",
       "                                         words_jieba  \n",
       "0  [好大;的;一个;游乐;公园;已经;去;了;2;次;但;感觉;还;没有;玩够;似的;！;会;...  \n",
       "1                                            [景色;秀丽]  \n",
       "2                              [辉煌;与;屈辱;也;都;成为;了;过去]  \n",
       "3  [东方明珠;广播;电视塔;是;上海;的;标志性;文化景观;之一;位于;浦东新区;陆家嘴;塔高...  \n",
       "4  [特色;的;建筑;各式;的;小店;好吃;的;小吃;还有;沙滩;和;朋友;和;家人;去;游玩;...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# snownlp 情感分析\n",
    "if not os.path.exists(model_path + 'senti_snownlp.csv'):\n",
    "    print('senti_snownlp file not exists...')\n",
    "    senti_snownlp_df = data[['Id', 'Discuss']].copy()\n",
    "    senti_snownlp_df['senti_snownlp'] = senti_snownlp_df['Discuss'].apply(lambda x : senti_snownlp(x))\n",
    "    del senti_snownlp_df['Discuss']\n",
    "    senti_snownlp_df.to_csv(model_path + 'senti_snownlp.csv', index = False)\n",
    "    data = pd.merge(data, senti_snownlp_df, on = 'Id', how = 'left')\n",
    "else:\n",
    "    print('senti_snownlp file exists...')\n",
    "    senti_snownlp_df = pd.read_csv(model_path + 'senti_snownlp.csv')\n",
    "    data = pd.merge(data, senti_snownlp_df, on = 'Id', how = 'left')\n",
    "\n",
    "    \n",
    "# 结巴分词\n",
    "if not os.path.exists(model_path + 'data_jieba.csv'):    \n",
    "    data_jieba = data[['Id', 'Discuss']].copy()\n",
    "    data_jieba['words_jieba'] = data_jieba['Discuss'].apply(lambda x : get_jieba_words(x))\n",
    "    del data_jieba['Discuss']\n",
    "    data_jieba.to_csv(model_path + 'data_jieba.csv', index = False, encoding = \"utf-8\")\n",
    "else:\n",
    "    print('data_jieba file exists...')\n",
    "    data_jieba = pd.read_csv(model_path + 'data_jieba.csv')\n",
    "\n",
    "data_jieba.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 130000 entries, 0 to 129999\n",
      "Data columns (total 5 columns):\n",
      "Id               130000 non-null object\n",
      "Discuss          130000 non-null object\n",
      "Score            130000 non-null int64\n",
      "senti_snownlp    130000 non-null float64\n",
      "strLen           130000 non-null int64\n",
      "dtypes: float64(1), int64(2), object(2)\n",
      "memory usage: 6.0+ MB\n",
      "None\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Discuss</th>\n",
       "      <th>Score</th>\n",
       "      <th>senti_snownlp</th>\n",
       "      <th>strLen</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>201e8bf2-77a2-3a98-9fcf-4ce03914e712</td>\n",
       "      <td>好大的一个游乐公园，已经去了2次，但感觉还没有玩够似的！会有第三，第四次的</td>\n",
       "      <td>5</td>\n",
       "      <td>0.521543</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f4d51947-eac4-3005-9d3c-2f32d6068a2d</td>\n",
       "      <td>新中国成立也是在这举行，对我们中国人来说有些重要及深刻的意义！</td>\n",
       "      <td>4</td>\n",
       "      <td>0.996441</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>74aa7ae4-03a4-394c-bee0-5702d3a3082a</td>\n",
       "      <td>庐山瀑布非常有名，也有非常多个瀑布，只是最好看的非三叠泉莫属，推荐一去</td>\n",
       "      <td>4</td>\n",
       "      <td>0.449366</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>099661c2-4360-3c49-a2fe-8c783764f7db</td>\n",
       "      <td>个人觉得颐和园是北京最值的一起的地方，不过相比下门票也是最贵的，比起故宫的雄伟与气势磅礴，颐...</td>\n",
       "      <td>5</td>\n",
       "      <td>0.985191</td>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>97ca672d-e558-3542-ba7b-ee719bba1bab</td>\n",
       "      <td>迪斯尼一日游</td>\n",
       "      <td>5</td>\n",
       "      <td>0.972308</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Id  \\\n",
       "0  201e8bf2-77a2-3a98-9fcf-4ce03914e712   \n",
       "1  f4d51947-eac4-3005-9d3c-2f32d6068a2d   \n",
       "2  74aa7ae4-03a4-394c-bee0-5702d3a3082a   \n",
       "3  099661c2-4360-3c49-a2fe-8c783764f7db   \n",
       "4  97ca672d-e558-3542-ba7b-ee719bba1bab   \n",
       "\n",
       "                                             Discuss  Score  senti_snownlp  \\\n",
       "0              好大的一个游乐公园，已经去了2次，但感觉还没有玩够似的！会有第三，第四次的      5       0.521543   \n",
       "1                    新中国成立也是在这举行，对我们中国人来说有些重要及深刻的意义！      4       0.996441   \n",
       "2                庐山瀑布非常有名，也有非常多个瀑布，只是最好看的非三叠泉莫属，推荐一去      4       0.449366   \n",
       "3  个人觉得颐和园是北京最值的一起的地方，不过相比下门票也是最贵的，比起故宫的雄伟与气势磅礴，颐...      5       0.985191   \n",
       "4                                             迪斯尼一日游      5       0.972308   \n",
       "\n",
       "   strLen  \n",
       "0      37  \n",
       "1      31  \n",
       "2      35  \n",
       "3      61  \n",
       "4       6  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 字符串的长度\n",
    "data['strLen'] = data['Discuss'].apply(lambda x : len(x))\n",
    "print(data.info())\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2005197\n",
      "[['UNK', 45072], ('人', 19245), ('不错', 17065), ('<br', 13191), ('一个', 12210), ('地方', 11896), ('好', 11706), ('景区', 11426), ('值得', 9536), ('景点', 8664), ('感觉', 8189), ('美', 7932), ('风景', 7747), ('比较', 6853), ('>rn', 6659), ('走', 6570), ('里面', 6434), ('景色', 6375), ('门票', 6349), ('次', 6283), ('玩', 6208), ('真的', 6034), ('天', 5995), ('看到', 5820), ('很好', 5529), ('元', 5450), ('时间', 5164), ('点', 4784), ('年', 4680), ('特别', 4625), ('说', 4578), ('北京', 4559), ('方便', 4516), ('最', 4498), ('看看', 4476), ('两', 4451), ('西湖', 4349), ('坐', 4266), ('小时', 4134), ('建筑', 4108), ('喜欢', 4096), ('小', 3969), ('太', 3909), ('下', 3857), ('一定', 3758), ('>n', 3721), ('里', 3696), ('历史', 3689), ('中国', 3671), ('觉得', 3657), ('后', 3622), ('中', 3508), ('建议', 3482), ('游客', 3395), ('适合', 3391), ('票', 3310), ('挺', 3268), ('旅游', 3253), ('晚上', 3231), ('第一', 3218), ('特色', 3208), ('位于', 3132), ('再', 3126), ('有点', 3078), ('推荐', 3073), ('很大', 3072), ('山', 2989), ('黄山', 2960), ('游', 2928), ('一下', 2926), ('吃', 2895), ('座', 2886), ('现在', 2873), ('公园', 2820), ('太多', 2794), ('游玩', 2778), ('米', 2713), ('需要', 2630), ('爬', 2572), ('杭州', 2532), ('想', 2531), ('导游', 2492), ('买', 2483), ('排队', 2465), ('环境', 2448), ('已经', 2444), ('壮观', 2374), ('索道', 2367), ('天气', 2357), ('文化', 2293), ('贵', 2220), ('知道', 2205), ('高', 2201), ('条', 2199), ('漂亮', 2179), ('山上', 2173), ('东西', 2170), ('水', 2150), ('一些', 2137), ('时', 2136), ('月', 2118), ('选择', 2110), ('世界', 2087), ('爬山', 2079), ('主要', 2075), ('好玩', 2064), ('逛', 2043), ('只', 2036), ('最好', 2020), ('累', 2015), ('一起', 2006), ('直接', 1982), ('游览', 1977), ('10', 1971), ('开心', 1964), ('做', 1914), ('泰山', 1906), ('拍照', 1891), ('应该', 1850), ('瀑布', 1849), ('拍', 1842), ('美丽', 1826), ('好多', 1825), ('孩子', 1803), ('整个', 1802), ('少', 1774), ('体验', 1770), ('故宫', 1762), ('朋友', 1761), ('一直', 1757), ('没什么', 1755), ('感受', 1732), ('震撼', 1703), ('山顶', 1691), ('种', 1674), ('缆车', 1669), ('美景', 1662), ('免费', 1660), ('日出', 1641), ('确实', 1627), ('第二', 1620), ('上山', 1614), ('表演', 1598), ('真是', 1591), ('参观', 1586), ('车', 1573), ('讲解', 1550), ('很漂亮', 1550), ('附近', 1541), ('一路', 1534), ('一点', 1527), ('号', 1524), ('赞', 1516), ('住', 1516), ('好看', 1503), ('只能', 1491), ('前', 1491), ('价格', 1484), ('门口', 1482), ('古城', 1475), ('路', 1471), ('挺好', 1470), ('之后', 1465), ('公里', 1449), ('下山', 1434), ('过去', 1427), ('好好', 1422), ('取', 1415), ('古镇', 1414), ('站', 1412), ('下来', 1402), ('最大', 1399), ('进入', 1396), ('分钟', 1395), ('长城', 1389), ('便宜', 1380), ('交通', 1378), ('空气', 1367), ('找', 1363), ('风光', 1361), ('服务', 1355), ('夏天', 1355), ('不用', 1353), ('可惜', 1347), ('项目', 1344), ('总体', 1342), ('进', 1342), ('点评', 1342), ('当时', 1332), ('20', 1327), ('话', 1317), ('30', 1315), ('花', 1299), ('下午', 1298), ('著名', 1294), ('左右', 1289), ('国家', 1278), ('来到', 1268), ('广场', 1262), ('日', 1261)]\n",
      "neg_words len:  11809\n",
      "[['UNK', 0], ('一个', 149), ('景区', 136), ('说', 133), ('人', 127), ('门票', 103), ('景点', 90), ('地方', 75), ('<br', 64), ('元', 60), ('里面', 57), ('小时', 56), ('两', 54), ('票', 53), ('买', 52), ('差', 51), ('坑', 46), ('点', 44), ('真的', 44), ('走', 44), ('太', 43), ('好', 43), ('感觉', 42), ('失望', 42), ('值得', 41), ('次', 40), ('坐', 39), ('知道', 39), ('没什么', 37), ('想', 37), ('后', 36), ('>n', 36), ('导游', 35), ('排队', 35), ('贵', 34), ('玩', 33), ('已经', 32), ('游客', 32), ('收费', 31), ('全', 31), ('最', 30), ('旅游', 30), ('工作人员', 29), ('再', 29), ('车', 29), ('差评', 29), ('张', 28), ('垃圾', 28), ('钱', 27), ('天', 27), ('管理', 27), ('一点', 26), ('东西', 26), ('根本', 26), ('收', 26), ('比较', 25), ('司机', 25), ('缆车', 25), ('建议', 24), ('看到', 24), ('花', 24), ('进', 24), ('只能', 24), ('问', 24), ('呵呵', 24), ('风景', 23), ('10', 23), ('完全', 23), ('买票', 23), ('之后', 22), ('小', 22), ('不错', 21), ('服务', 20), ('不值', 20), ('吃', 20), ('第二', 20), ('下', 20), ('网上', 20), ('特别', 20), ('这种', 20), ('真是', 20), ('一下', 20), ('还要', 19), ('块', 19), ('买了', 19), ('坑人', 19), ('觉得', 19), ('时间', 18), ('直接', 18), ('卖', 18), ('开', 18), ('里', 18), ('丽江', 18), ('古城', 18), ('20', 17), ('条', 17), ('做', 17), ('号', 17), ('以后', 17), ('真', 17)]\n",
      "[['UNK', 96482], ('不错', 4135), ('值得', 2534), ('好', 2342), ('美', 1851), ('地方', 1777), ('很好', 1705), ('人', 1561), ('风景', 1434), ('玩', 1387), ('方便', 1355), ('景色', 1241), ('次', 1141), ('景点', 984), ('游', 930), ('喜欢', 892), ('开心', 879), ('好玩', 850), ('真的', 823), ('适合', 819), ('西湖', 818), ('感觉', 784), ('一个', 783), ('黄山', 704), ('景区', 704), ('特别', 689), ('漂亮', 671), ('天', 626), ('环境', 624), ('比较', 614), ('一定', 604), ('看看', 569), ('壮观', 569), ('北京', 567), ('游玩', 566), ('票', 556), ('孩子', 552), ('美丽', 530), ('<br', 522), ('推荐', 517), ('挺好', 515), ('很漂亮', 510), ('出游', 509), ('挺', 503), ('很大', 492), ('太多', 488), ('亲子游', 487), ('迪士尼', 485), ('有奖', 485), ('里面', 482), ('赞', 481), ('旅行', 478), ('历史', 477), ('太', 474), ('震撼', 466), ('很棒', 466), ('特色', 459), ('取', 457), ('累', 449), ('季', 446), ('棒', 445), ('排队', 444), ('三清山', 438), ('泰山', 430), ('下次', 424), ('看到', 417), ('中国', 413), ('体验', 406), ('门票', 405), ('点评', 404), ('旅游', 396), ('最', 393), ('时间', 392), ('再', 385), ('空气', 384), ('之旅', 382), ('建筑', 381), ('故宫', 378), ('爬', 372), ('杭州', 371), ('走', 365), ('天气', 364), ('有点', 363), ('超级', 359), ('晚上', 357), ('一日游', 355), ('日出', 351), ('>rn', 349), ('索道', 349), ('刺激', 330), ('满意', 330), ('好看', 326), ('值得一看', 326), ('小', 325), ('美景', 324), ('感受', 320), ('>n', 319), ('世界', 317), ('爬山', 316), ('文化', 313)]\n",
      "[['UNK', 66867], ('不错', 1720), ('值得', 879), ('人', 855), ('地方', 685), ('风景', 580), ('好', 547), ('景色', 521), ('比较', 517), ('景点', 514), ('美', 500), ('感觉', 474), ('适合', 450), ('看看', 442), ('玩', 421), ('景区', 387), ('<br', 375), ('太多', 366), ('一个', 365), ('很好', 364), ('游', 350), ('挺', 345), ('里面', 338), ('特色', 318), ('次', 318), ('北京', 309), ('西湖', 294), ('还行', 288), ('公园', 283), ('喜欢', 280), ('门票', 279), ('建筑', 277), ('有点', 275), ('很大', 273), ('晚上', 261), ('>n', 258), ('总体', 255), ('环境', 250), ('真的', 249), ('历史', 248), ('太', 242), ('方便', 240), ('特别', 234), ('走', 232), ('贵', 229), ('>rn', 215), ('挺好', 212), ('小', 210), ('看到', 205), ('时间', 204), ('游玩', 197), ('孩子', 191), ('很漂亮', 191), ('逛', 190), ('元', 189), ('壮观', 187), ('天', 185), ('好看', 179), ('一定', 179), ('点', 177), ('天气', 177), ('性价比', 176), ('古城', 169), ('表演', 168), ('吃', 168), ('排队', 167), ('没什么', 167), ('拍照', 167), ('一下', 167), ('故宫', 166), ('好玩', 165), ('现在', 164), ('东西', 163), ('项目', 163), ('夏天', 163), ('觉得', 162), ('夜景', 161), ('少', 161), ('漂亮', 160), ('高', 158), ('般', 157), ('商业化', 157), ('好多', 156), ('推荐', 155), ('开心', 154), ('古镇', 154), ('应该', 153), ('杭州', 152), ('中国', 152), ('累', 151), ('出游', 151), ('爬山', 147), ('导游', 146), ('空气', 145), ('山', 144), ('票', 143), ('旅游', 143), ('还好', 142), ('爬', 142), ('风光', 142)]\n",
      "[['UNK', 16291], ('不错', 374), ('人', 327), ('感觉', 303), ('地方', 265), ('景点', 264), ('没什么', 243), ('好', 240), ('一个', 207), ('看看', 192), ('般', 187), ('风景', 183), ('比较', 182), ('景色', 165), ('太', 164), ('太多', 157), ('门票', 155), ('里面', 153), ('特色', 149), ('值得', 148), ('公园', 139), ('景区', 131), ('<br', 130), ('玩', 128), ('特别', 123), ('贵', 121), ('有点', 115), ('还行', 110), ('适合', 110), ('小', 110), ('性价比', 109), ('觉得', 104), ('商业化', 103), ('真的', 103), ('没啥', 101), ('挺', 100), ('高', 96), ('美', 94), ('>n', 93), ('失望', 92), ('建筑', 92), ('路过', 92), ('现在', 92), ('意思', 90), ('历史', 90), ('东西', 87), ('次', 86), ('喜欢', 86), ('很大', 84), ('好看', 82), ('西湖', 80), ('晚上', 79), ('看到', 79), ('很好', 78), ('北京', 78), ('好玩', 77), ('走', 74), ('推荐', 73), ('一下', 73), ('不好', 71), ('时间', 71), ('古镇', 69), ('>rn', 67), ('说', 67), ('想象', 67), ('商业', 67), ('元', 67), ('排队', 66), ('外面', 66), ('环境', 65), ('条', 64), ('寺庙', 63), ('已经', 63), ('旁边', 62), ('天气', 58), ('少', 58), ('气息', 58), ('逛', 57), ('两', 57), ('附近', 57), ('夏天', 57), ('普通', 56), ('知道', 56), ('山', 56), ('导游', 54), ('下', 53), ('逛逛', 53), ('建议', 53), ('广场', 52), ('游玩', 52), ('表演', 52), ('吃', 52), ('后悔', 52), ('中国', 52), ('方便', 52), ('只', 51), ('还好', 51), ('拍照', 51), ('完全', 51), ('水', 51)]\n",
      "[['UNK', 0], ('没什么', 50), ('人', 41), ('景点', 41), ('地方', 36), ('门票', 33), ('推荐', 31), ('一个', 30), ('贵', 25), ('意思', 25), ('太', 24), ('感觉', 24), ('没啥', 22), ('性价比', 22), ('好', 22), ('景区', 21), ('不好', 20), ('坑', 20), ('特色', 20), ('失望', 19), ('太多', 19), ('玩', 19), ('差', 18), ('东西', 18), ('般', 17), ('无聊', 17), ('有点', 17), ('里面', 17), ('商业', 16), ('好看', 16), ('比较', 16), ('真心', 15), ('值得', 15), ('不错', 15), ('觉得', 15), ('不值', 14), ('可看', 14), ('特别', 14), ('实在', 13), ('项目', 13), ('少', 13), ('没意思', 12), ('建议', 12), ('卖', 12), ('不好玩', 11), ('元', 11), ('服务', 11), ('普通', 11), ('挺', 11), ('还行', 10), ('排队', 10), ('次', 10), ('路过', 10), ('必要', 10), ('票价', 10), ('公园', 10), ('花', 10), ('小', 10), ('一些', 10), ('收费', 9), ('收', 9), ('风景', 9), ('建筑', 9), ('现在', 9), ('人造', 8), ('后悔', 8), ('>n', 8), ('块', 8), ('高', 8), ('古城', 8), ('<br', 8), ('所谓', 8), ('说', 8), ('名气', 8), ('一点', 8), ('还要', 8), ('适合', 8), ('景观', 8), ('完全', 8), ('全', 8), ('想', 8), ('再', 8), ('旅游', 8), ('古镇', 7), ('岛', 7), ('电梯', 7), ('钱', 7), ('门口', 7), ('商业街', 7), ('说实话', 7), ('年', 7), ('做', 7), ('价格', 7), ('人工', 7), ('点', 7), ('味道', 7), ('吃', 7), ('真', 7), ('装修', 6), ('消费', 6)]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['UNK', 0], ('值得', 20), ('失望', 20), ('没什么', 18), ('差', 18), ('景区', 18), ('门票', 18), ('坑', 17), ('不好玩', 16), ('差评', 15), ('不值', 14), ('垃圾', 13), ('次', 13), ('说', 12), ('景点', 11), ('人', 10), ('不想', 10), ('坑人', 10), ('太', 10), ('根本', 10), ('退', 9), ('退票', 9), ('司机', 9), ('意思', 9), ('没啥', 9), ('收费', 9), ('小时', 9), ('推荐', 9), ('全', 9), ('里面', 9), ('后悔', 8), ('骗子', 8), ('消费', 8), ('排队', 8), ('坑爹', 8), ('买', 8), ('一点', 8), ('知道', 8), ('元', 8), ('太差', 7), ('导游', 7), ('千万别', 7), ('评论', 7), ('贵', 7), ('收', 7), ('这种', 7), ('完全', 7), ('<br', 7), ('地方', 7), ('一个', 7), ('东西', 7), ('无聊', 6), ('黑', 6), ('钱', 6), ('太贵', 6), ('再也', 6), ('票', 6), ('投诉', 6), ('服务', 6), ('字', 6), ('缆车', 6), ('好', 6), ('再', 6), ('车', 6), ('真心', 6), ('特色', 6), ('破', 5), ('糟糕', 5), ('极差', 5), ('没意思', 5), ('>n', 5), ('上当', 5), ('最差', 5), ('团', 5), ('太多', 5), ('酒店', 5), ('买票', 5), ('工作人员', 5), ('票价', 5), ('烂', 5), ('感觉', 5), ('不敢', 5), ('体验', 5), ('停车费', 5), ('浪费时间', 5), ('特别', 5), ('山上', 5), ('买了', 5), ('骗', 5), ('比较', 5), ('不错', 5), ('小', 5), ('携程', 5), ('后', 5), ('设施', 5), ('已经', 5), ('想', 5), ('只', 5), ('少', 5), ('退款', 4)]\n",
      "2124625\n",
      "[['UNK', 55892], (' ', 72410), ('好', 19048), ('不错', 17257), ('br', 15442), ('人', 15226), ('一个', 12283), ('地方', 12238), ('景区', 12129), ('值得', 9652), ('景点', 8966), ('感觉', 8526), ('风景', 8448), ('走', 7899), ('rn', 7466), ('比较', 7247), ('美', 6660), ('里面', 6581), ('门票', 6464), ('景色', 6431), ('真的', 6185), ('看到', 5918), ('说', 5574), ('元', 5414), ('时间', 5227), ('北京', 5084), ('玩', 5074), ('特别', 4888), ('西湖', 4781), ('方便', 4780), ('最', 4759), ('中', 4722), ('喜欢', 4635), ('看看', 4624), ('挺', 4623), ('坐', 4373), ('建筑', 4204), ('中国', 4155), ('小', 4153), ('下', 4135), ('n', 4076), ('一定', 4020), ('再', 3981), ('爬', 3829), ('历史', 3819), ('小时', 3790), ('里', 3757), ('一去', 3756), ('觉得', 3675), ('年', 3673), (',', 3653), ('旅游', 3590), ('公园', 3570), ('建议', 3551), ('后', 3477), ('游客', 3455), ('适合', 3426), ('黄山', 3419), ('太', 3392), ('买', 3373), ('有点', 3334), ('特色', 3304), ('晚上', 3289), ('吃', 3280), ('位于', 3222), ('很大', 3211), ('想', 3114), ('推荐', 3104), ('一下', 3061), ('一次', 3051), ('现在', 2897), ('排队', 2859), ('游玩', 2835), ('需要', 2673), ('杭州', 2612), ('高', 2601), ('导游', 2537), ('索道', 2527), ('文化', 2507), ('已经', 2501), ('好玩', 2495), ('环境', 2474), ('游', 2458), ('路', 2449), ('壮观', 2397), ('月', 2395), ('主要', 2388), ('点', 2372), ('山', 2338), ('世界', 2331), ('爬山', 2301), ('天气', 2288), ('米', 2285), ('山上', 2283), ('东西', 2256), ('感受', 2248), ('知道', 2244), ('住', 2232), ('一些', 2222), ('一天', 2196), ('选择', 2161), ('逛', 2159), ('泰山', 2116), ('漂亮', 2114), ('站', 2103), ('朋友', 2065), ('10', 2044), ('贵', 2037), ('一起', 2023), ('直接', 2022), ('游览', 2019), ('古城', 2013), ('开心', 2006), ('最好', 2000), ('瀑布', 1984), ('孩子', 1961), ('时', 1935), ('故宫', 1932), ('拍照', 1925), ('美丽', 1905), ('水', 1896), ('应该', 1872), ('做', 1843), ('拍', 1842), ('日出', 1812), ('整个', 1807), ('】', 1800), ('体验', 1799), ('【', 1798), ('一直', 1791), ('没什么', 1790), ('完', 1781), ('上山', 1779), ('只', 1759), ('好多', 1755), ('去过', 1738), ('山顶', 1732), ('缆车', 1724), ('震撼', 1711), ('长城', 1700), ('美景', 1694), ('免费', 1680), ('博物馆', 1657), ('确实', 1644), ('累', 1642), ('表演', 1631), ('参观', 1616), ('下山', 1588), ('票', 1579), ('讲解', 1571), ('附近', 1563), ('很漂亮', 1559), ('一路', 1554), ('两个', 1552), ('一座', 1542), ('只能', 1542), ('价格', 1540), ('国家', 1532), ('真是', 1525), ('广场', 1523), ('门口', 1522), ('之后', 1502), ('行', 1501), ('古镇', 1496), ('好看', 1496), ('过去', 1487), ('第一次', 1463), ('好好', 1458), ('一条', 1456), ('最大', 1439), ('下来', 1434), ('一点', 1433), ('进入', 1432), ('交通', 1427), ('风光', 1424), ('人太多', 1419), ('夏天', 1416), ('便宜', 1408), ('不用', 1407), ('项目', 1395), ('空气', 1391), ('下午', 1391), ('当时', 1387), ('服务', 1386), ('可惜', 1368), ('30', 1366), ('点评', 1357), ('超级', 1354), ('亲子', 1352), ('总体', 1352), ('20', 1347), ('著名', 1322), ('前', 1320), ('到达', 1304), ('来到', 1282), ('听', 1276), ('人多', 1270), ('\\xa0', 1260), ('性价比', 1255), ('之前', 1254)]\n",
      "neg_words len:  12273\n",
      "[['UNK', 0], (' ', 418), ('说', 153), ('景区', 143), ('一个', 141), ('人', 119), ('门票', 104), ('景点', 91), ('地方', 75), ('br', 70), ('买', 68), ('里面', 58), ('元', 55), ('走', 55), ('差', 51), ('排队', 50), ('太', 49), ('好', 48), ('小时', 47), ('想', 46), ('真的', 44), ('感觉', 44), ('坐', 44), ('失望', 42), ('值得', 41), ('票', 40), ('知道', 39), ('旅游', 37), ('n', 37), ('没什么', 37), ('导游', 35), ('再', 35), ('坑', 35), ('钱', 34), ('已经', 32), ('玩', 32), ('游客', 32), ('收费', 32), ('两个', 31), ('工作人员', 29), ('车', 28), ('后', 27), ('管理', 27), ('垃圾', 27), ('缆车', 27), ('一点', 26), ('比较', 26), ('东西', 26), ('根本', 26), ('司机', 25), ('贵', 25), ('呵呵', 24), ('建议', 24), ('看到', 24), ('只能', 24), ('丽江', 24), ('一次', 24), ('买票', 23), ('小', 23), ('差评', 23), ('风景', 23), ('10', 23), ('完全', 23), ('收', 23), ('一下', 23), ('进', 22), ('服务', 22), ('之后', 22), ('最', 22), ('里', 22), ('特别', 21), ('不错', 21), ('古城', 21), ('花', 21), ('网上', 20), ('这种', 20), ('吃', 20), ('还要', 20), ('点', 19), ('觉得', 19), ('全是', 19), ('20', 19), ('坑人', 19), ('卖', 19), ('千万别', 18), ('水', 18), ('直接', 18), ('不好', 18), ('索道', 18), ('不值', 18), ('酒店', 18), ('下', 18), ('真是', 18), ('不好玩', 18), ('时间', 17), ('门口', 17), ('实在', 17), ('号', 17), ('北京', 17), ('做', 17)]\n",
      "[['UNK', 96482], ('不错', 4135), ('值得', 2534), ('好', 2342), ('美', 1851), ('地方', 1777), ('很好', 1705), ('人', 1561), ('风景', 1434), ('玩', 1387), ('方便', 1355), ('景色', 1241), ('次', 1141), ('景点', 984), ('游', 930), ('喜欢', 892), ('开心', 879), ('好玩', 850), ('真的', 823), ('适合', 819), ('西湖', 818), ('感觉', 784), ('一个', 783), ('黄山', 704), ('景区', 704), ('特别', 689), ('漂亮', 671), ('天', 626), ('环境', 624), ('比较', 614), ('一定', 604), ('看看', 569), ('壮观', 569), ('北京', 567), ('游玩', 566), ('票', 556), ('孩子', 552), ('美丽', 530), ('<br', 522), ('推荐', 517), ('挺好', 515), ('很漂亮', 510), ('出游', 509), ('挺', 503), ('很大', 492), ('太多', 488), ('亲子游', 487), ('迪士尼', 485), ('有奖', 485), ('里面', 482), ('赞', 481), ('旅行', 478), ('历史', 477), ('太', 474), ('震撼', 466), ('很棒', 466), ('特色', 459), ('取', 457), ('累', 449), ('季', 446), ('棒', 445), ('排队', 444), ('三清山', 438), ('泰山', 430), ('下次', 424), ('看到', 417), ('中国', 413), ('体验', 406), ('门票', 405), ('点评', 404), ('旅游', 396), ('最', 393), ('时间', 392), ('再', 385), ('空气', 384), ('之旅', 382), ('建筑', 381), ('故宫', 378), ('爬', 372), ('杭州', 371), ('走', 365), ('天气', 364), ('有点', 363), ('超级', 359), ('晚上', 357), ('一日游', 355), ('日出', 351), ('>rn', 349), ('索道', 349), ('刺激', 330), ('满意', 330), ('好看', 326), ('值得一看', 326), ('小', 325), ('美景', 324), ('感受', 320), ('>n', 319), ('世界', 317), ('爬山', 316), ('文化', 313)]\n",
      "[['UNK', 66867], ('不错', 1720), ('值得', 879), ('人', 855), ('地方', 685), ('风景', 580), ('好', 547), ('景色', 521), ('比较', 517), ('景点', 514), ('美', 500), ('感觉', 474), ('适合', 450), ('看看', 442), ('玩', 421), ('景区', 387), ('<br', 375), ('太多', 366), ('一个', 365), ('很好', 364), ('游', 350), ('挺', 345), ('里面', 338), ('特色', 318), ('次', 318), ('北京', 309), ('西湖', 294), ('还行', 288), ('公园', 283), ('喜欢', 280), ('门票', 279), ('建筑', 277), ('有点', 275), ('很大', 273), ('晚上', 261), ('>n', 258), ('总体', 255), ('环境', 250), ('真的', 249), ('历史', 248), ('太', 242), ('方便', 240), ('特别', 234), ('走', 232), ('贵', 229), ('>rn', 215), ('挺好', 212), ('小', 210), ('看到', 205), ('时间', 204), ('游玩', 197), ('孩子', 191), ('很漂亮', 191), ('逛', 190), ('元', 189), ('壮观', 187), ('天', 185), ('好看', 179), ('一定', 179), ('点', 177), ('天气', 177), ('性价比', 176), ('古城', 169), ('表演', 168), ('吃', 168), ('排队', 167), ('没什么', 167), ('拍照', 167), ('一下', 167), ('故宫', 166), ('好玩', 165), ('现在', 164), ('东西', 163), ('项目', 163), ('夏天', 163), ('觉得', 162), ('夜景', 161), ('少', 161), ('漂亮', 160), ('高', 158), ('般', 157), ('商业化', 157), ('好多', 156), ('推荐', 155), ('开心', 154), ('古镇', 154), ('应该', 153), ('杭州', 152), ('中国', 152), ('累', 151), ('出游', 151), ('爬山', 147), ('导游', 146), ('空气', 145), ('山', 144), ('票', 143), ('旅游', 143), ('还好', 142), ('爬', 142), ('风光', 142)]\n",
      "[['UNK', 16291], ('不错', 374), ('人', 327), ('感觉', 303), ('地方', 265), ('景点', 264), ('没什么', 243), ('好', 240), ('一个', 207), ('看看', 192), ('般', 187), ('风景', 183), ('比较', 182), ('景色', 165), ('太', 164), ('太多', 157), ('门票', 155), ('里面', 153), ('特色', 149), ('值得', 148), ('公园', 139), ('景区', 131), ('<br', 130), ('玩', 128), ('特别', 123), ('贵', 121), ('有点', 115), ('还行', 110), ('适合', 110), ('小', 110), ('性价比', 109), ('觉得', 104), ('商业化', 103), ('真的', 103), ('没啥', 101), ('挺', 100), ('高', 96), ('美', 94), ('>n', 93), ('失望', 92), ('建筑', 92), ('路过', 92), ('现在', 92), ('意思', 90), ('历史', 90), ('东西', 87), ('次', 86), ('喜欢', 86), ('很大', 84), ('好看', 82), ('西湖', 80), ('晚上', 79), ('看到', 79), ('很好', 78), ('北京', 78), ('好玩', 77), ('走', 74), ('推荐', 73), ('一下', 73), ('不好', 71), ('时间', 71), ('古镇', 69), ('>rn', 67), ('说', 67), ('想象', 67), ('商业', 67), ('元', 67), ('排队', 66), ('外面', 66), ('环境', 65), ('条', 64), ('寺庙', 63), ('已经', 63), ('旁边', 62), ('天气', 58), ('少', 58), ('气息', 58), ('逛', 57), ('两', 57), ('附近', 57), ('夏天', 57), ('普通', 56), ('知道', 56), ('山', 56), ('导游', 54), ('下', 53), ('逛逛', 53), ('建议', 53), ('广场', 52), ('游玩', 52), ('表演', 52), ('吃', 52), ('后悔', 52), ('中国', 52), ('方便', 52), ('只', 51), ('还好', 51), ('拍照', 51), ('完全', 51), ('水', 51)]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['UNK', 0], ('没什么', 50), ('人', 41), ('景点', 41), ('地方', 36), ('门票', 33), ('推荐', 31), ('一个', 30), ('贵', 25), ('意思', 25), ('太', 24), ('感觉', 24), ('没啥', 22), ('性价比', 22), ('好', 22), ('景区', 21), ('不好', 20), ('坑', 20), ('特色', 20), ('失望', 19), ('太多', 19), ('玩', 19), ('差', 18), ('东西', 18), ('般', 17), ('无聊', 17), ('有点', 17), ('里面', 17), ('商业', 16), ('好看', 16), ('比较', 16), ('真心', 15), ('值得', 15), ('不错', 15), ('觉得', 15), ('不值', 14), ('可看', 14), ('特别', 14), ('实在', 13), ('项目', 13), ('少', 13), ('没意思', 12), ('建议', 12), ('卖', 12), ('不好玩', 11), ('元', 11), ('服务', 11), ('普通', 11), ('挺', 11), ('还行', 10), ('排队', 10), ('次', 10), ('路过', 10), ('必要', 10), ('票价', 10), ('公园', 10), ('花', 10), ('小', 10), ('一些', 10), ('收费', 9), ('收', 9), ('风景', 9), ('建筑', 9), ('现在', 9), ('人造', 8), ('后悔', 8), ('>n', 8), ('块', 8), ('高', 8), ('古城', 8), ('<br', 8), ('所谓', 8), ('说', 8), ('名气', 8), ('一点', 8), ('还要', 8), ('适合', 8), ('景观', 8), ('完全', 8), ('全', 8), ('想', 8), ('再', 8), ('旅游', 8), ('古镇', 7), ('岛', 7), ('电梯', 7), ('钱', 7), ('门口', 7), ('商业街', 7), ('说实话', 7), ('年', 7), ('做', 7), ('价格', 7), ('人工', 7), ('点', 7), ('味道', 7), ('吃', 7), ('真', 7), ('装修', 6), ('消费', 6)]\n",
      "[['UNK', 0], ('值得', 20), ('失望', 20), ('没什么', 18), ('差', 18), ('景区', 18), ('门票', 18), ('坑', 17), ('不好玩', 16), ('差评', 15), ('不值', 14), ('垃圾', 13), ('次', 13), ('说', 12), ('景点', 11), ('人', 10), ('不想', 10), ('坑人', 10), ('太', 10), ('根本', 10), ('退', 9), ('退票', 9), ('司机', 9), ('意思', 9), ('没啥', 9), ('收费', 9), ('小时', 9), ('推荐', 9), ('全', 9), ('里面', 9), ('后悔', 8), ('骗子', 8), ('消费', 8), ('排队', 8), ('坑爹', 8), ('买', 8), ('一点', 8), ('知道', 8), ('元', 8), ('太差', 7), ('导游', 7), ('千万别', 7), ('评论', 7), ('贵', 7), ('收', 7), ('这种', 7), ('完全', 7), ('<br', 7), ('地方', 7), ('一个', 7), ('东西', 7), ('无聊', 6), ('黑', 6), ('钱', 6), ('太贵', 6), ('再也', 6), ('票', 6), ('投诉', 6), ('服务', 6), ('字', 6), ('缆车', 6), ('好', 6), ('再', 6), ('车', 6), ('真心', 6), ('特色', 6), ('破', 5), ('糟糕', 5), ('极差', 5), ('没意思', 5), ('>n', 5), ('上当', 5), ('最差', 5), ('团', 5), ('太多', 5), ('酒店', 5), ('买票', 5), ('工作人员', 5), ('票价', 5), ('烂', 5), ('感觉', 5), ('不敢', 5), ('体验', 5), ('停车费', 5), ('浪费时间', 5), ('特别', 5), ('山上', 5), ('买了', 5), ('骗', 5), ('比较', 5), ('不错', 5), ('小', 5), ('携程', 5), ('后', 5), ('设施', 5), ('已经', 5), ('想', 5), ('只', 5), ('少', 5), ('退款', 4)]\n"
     ]
    }
   ],
   "source": [
    "def data_word_feature(data_word, feature_word = 'words'):\n",
    "    # 去除停用词\n",
    "    stop_list = read_file_word2set('../input/stop_word.txt')\n",
    "    \n",
    "    # all words\n",
    "    dictionary = get_words(data_word, stop_list, feature_word)\n",
    "    print(len(dictionary))\n",
    "    word_num, count, dictionary, reverse_dictionary = build_dataset(dictionary, vocabulary_size=50000)\n",
    "    print(count[:200])\n",
    "\n",
    "    # negative words\n",
    "    neg_words = get_neg_words(pd.merge(data_word, data[['Id', 'Score']], on = 'Id', how = 'left'), stop_list, feature_word, 'Score')\n",
    "    print('neg_words len: ', len(neg_words))\n",
    "    word_num_neg, count_neg, dictionary_neg, reverse_dictionary_neg = build_dataset(neg_words, vocabulary_size=5000)\n",
    "    print(count_neg[:100])\n",
    "\n",
    "    # 群里siginicant word\n",
    "    significant_dictionary =  read_file_word2dict(model_path + 'significance.txt')\n",
    "        \n",
    "    # encoding significant words\n",
    "    def significant_word(word, dictionary = significant_dictionary):\n",
    "        sent = word[1:-1]\n",
    "        words = sent.split(';')\n",
    "        encoding = 0.0\n",
    "        arra = []\n",
    "        for word in words:\n",
    "            if word in dictionary:\n",
    "                arra.append(dictionary[word])\n",
    "        if len(arra) == 0: return 0\n",
    "        arra = sorted(arra)\n",
    "        for a in arra:\n",
    "            encoding = 1.0 * len(dictionary) * encoding + a\n",
    "        return encoding\n",
    "\n",
    "    # neg_words 出现的总数\n",
    "    def neg_words_sum(word, dictionary = dictionary_neg):\n",
    "        sent = word[1:-1]\n",
    "        words = sent.split(';')\n",
    "        cnt = 0\n",
    "        for word in words:\n",
    "            if word in dictionary: cnt += 1\n",
    "        return cnt\n",
    "\n",
    "    # neg_words 出现的频次\n",
    "    def neg_words_ratio(word, dictionary = dictionary_neg):\n",
    "        sent = word[1:-1]\n",
    "        words = sent.split(';')\n",
    "        if len(words) == 0: return ''\n",
    "        cnt = 0\n",
    "        for word in words:\n",
    "            if word in dictionary: cnt += 1\n",
    "        return cnt / len(words)\n",
    "\n",
    "\n",
    "    def words_to_number(word, dictionary):\n",
    "        sent = word[1:-1]\n",
    "        words = sent.split(';')\n",
    "        data = []\n",
    "        for word in words:\n",
    "            if word in dictionary: data.append(dictionary[word])\n",
    "            else: data.append(0)\n",
    "        return data\n",
    "\n",
    "    def words_to_sum(word, dictionary):\n",
    "        sent = word[1:-1]\n",
    "        words = sent.split(';')\n",
    "        data = []\n",
    "        for word in words:\n",
    "            if word in dictionary: data.append(dictionary[word])\n",
    "            else: data.append(0)\n",
    "        return np.sum(data)\n",
    "\n",
    "    def words_to_max(word, dictionary):\n",
    "        sent = word[1:-1]\n",
    "        words = sent.split(';')\n",
    "        data = []\n",
    "        for word in words:\n",
    "            if word in dictionary.keys(): \n",
    "                data.append(dictionary[word])\n",
    "            else: data.append(0)\n",
    "        return np.max(data)\n",
    "\n",
    "    def words_to_std(word, dictionary):\n",
    "        sent = word[1:-1]\n",
    "        words = sent.split(';')\n",
    "        data = []\n",
    "        for word in words:\n",
    "            if word in dictionary: data.append(dictionary[word])\n",
    "            else: data.append(0)\n",
    "        return np.std(data)\n",
    "\n",
    "    def words_to_last(word, dictionary):\n",
    "        sent = word[1:-1]\n",
    "        words = sent.split(';')\n",
    "        if len(words) == 0: return ''\n",
    "        return dictionary[words[0]] if words[0] in dictionary else 0\n",
    "\n",
    "\n",
    "    def words_to_first(word, dictionary):\n",
    "        sent = word[1:-1]\n",
    "        words = sent.split(';')\n",
    "        if len(words) == 0: return ''\n",
    "        return dictionary[words[-1]] if words[-1] in dictionary else 0\n",
    "\n",
    "    def words_len(word):\n",
    "        sent = word[1:-1]\n",
    "        words = sent.split(';')\n",
    "        return len(words)\n",
    "    \n",
    "    def hasNtusdPos(word):\n",
    "        sent = word[1:-1]\n",
    "        words = sent.split(';')\n",
    "        if len(words) == 0: return ''\n",
    "        for word in words:\n",
    "            if word in ntusd_pos:\n",
    "                return 1\n",
    "        return 0\n",
    "    \n",
    "    def hasNtusdNum(word, ntusd):\n",
    "        sent = word[1:-1]\n",
    "        words = sent.split(';')\n",
    "        if len(words) == 0: return 0\n",
    "        cnt = 0\n",
    "        for word in words:\n",
    "            if word in ntusd:\n",
    "                cnt += 1\n",
    "        return cnt\n",
    "    \n",
    "    def hasNtusdNeg(word):\n",
    "        sent = word[1:-1]\n",
    "        words = sent.split(';')\n",
    "        if len(words) == 0: return ''\n",
    "        for word in words:\n",
    "            if word in ntusd_neg:\n",
    "                return 1\n",
    "        return 0\n",
    "    \n",
    "    data_word['word_len_'+ feature_word] = data_word[feature_word].apply(lambda x : words_len(x))\n",
    "    data_word['word_max_'+ feature_word] = data_word[feature_word].apply(lambda x : words_to_max(x, dictionary))\n",
    "    data_word['word_std_'+ feature_word] = data_word[feature_word].apply(lambda x : words_to_std(x, dictionary))\n",
    "    data_word['word_sum_'+ feature_word] = data_word[feature_word].apply(lambda x : words_to_sum(x, dictionary))\n",
    "    data_word['word_last_'+ feature_word] = data_word[feature_word].apply(lambda x : words_to_last(x, dictionary))\n",
    "    data_word['word_first_'+ feature_word] = data_word[feature_word].apply(lambda x : words_to_first(x, dictionary))\n",
    "    data_word['neg_word_sum_'+ feature_word] = data_word[feature_word].apply(lambda x : neg_words_sum(x, dictionary_neg))\n",
    "    data_word['neg_word_ratio_'+ feature_word] = data_word[feature_word].apply(lambda x : neg_words_ratio(x, dictionary_neg))\n",
    "    data_word['significant_word_encoding_'+ feature_word] = data_word[feature_word].apply(lambda x : significant_word(x, significant_dictionary))\n",
    "        \n",
    "    # ntusd_pos\n",
    "    data_word['word_ntusd_hasPos'+ feature_word] = data_word[feature_word].apply(lambda x : hasNtusdPos(x))\n",
    "    data_word['word_ntusd_hasNeg'+ feature_word] = data_word[feature_word].apply(lambda x : hasNtusdNeg(x))\n",
    "    data_word['word_ntusd_hasPosNum'+ feature_word] = data_word[feature_word].apply(lambda x : hasNtusdNum(x, ntusd_pos))\n",
    "    data_word['word_ntusd_hasNegNum'+ feature_word] = data_word[feature_word].apply(lambda x : hasNtusdNum(x, ntusd_neg))\n",
    "    \n",
    "    # get key word dictionary\n",
    "    dict_5 = getdictionary(topK = 10, stop_set = stop_list, label = 5, maxLen = 5000)\n",
    "    data_word['word_max_'+ feature_word + 'score_5'] = data_word[feature_word].apply(lambda x : words_to_max(x, dict_5))\n",
    "    data_word['word_std_'+ feature_word + 'score_5'] = data_word[feature_word].apply(lambda x : words_to_std(x, dict_5))\n",
    "    data_word['word_sum_'+ feature_word + 'score_5'] = data_word[feature_word].apply(lambda x : words_to_sum(x, dict_5))\n",
    "    data_word['word_last_'+ feature_word + 'score_5'] = data_word[feature_word].apply(lambda x : words_to_last(x,dict_5))\n",
    "    data_word['word_first_'+ feature_word + 'score_5'] = data_word[feature_word].apply(lambda x : words_to_first(x, dict_5))\n",
    "    \n",
    "    dict_4 = getdictionary(topK = 10, stop_set = stop_list, label = 4, maxLen = 5000)\n",
    "    data_word['word_max_'+ feature_word + 'score_4'] = data_word[feature_word].apply(lambda x : words_to_max(x, dict_4))\n",
    "    data_word['word_std_'+ feature_word + 'score_4'] = data_word[feature_word].apply(lambda x : words_to_std(x, dict_4))\n",
    "    data_word['word_sum_'+ feature_word + 'score_4'] = data_word[feature_word].apply(lambda x : words_to_sum(x, dict_4))\n",
    "    data_word['word_last_'+ feature_word + 'score_4'] = data_word[feature_word].apply(lambda x : words_to_last(x,dict_4))\n",
    "    data_word['word_first_'+ feature_word + 'score_4'] = data_word[feature_word].apply(lambda x : words_to_first(x, dict_4))\n",
    "    \n",
    "    dict_3 = getdictionary(topK = 10, stop_set = stop_list, label = 3, maxLen = 5000)\n",
    "    data_word['word_std_'+ feature_word + 'score_3'] = data_word[feature_word].apply(lambda x : words_to_std(x, dict_3))\n",
    "    data_word['word_sum_'+ feature_word + 'score_3'] = data_word[feature_word].apply(lambda x : words_to_sum(x, dict_3))\n",
    "    data_word['word_last_'+ feature_word + 'score_3'] = data_word[feature_word].apply(lambda x : words_to_last(x,dict_3))\n",
    "    data_word['word_first_'+ feature_word + 'score_3'] = data_word[feature_word].apply(lambda x : words_to_first(x, dict_3))\n",
    "    \n",
    "    dict_2 = getdictionary(topK = 10, stop_set = stop_list, label = 2, maxLen = 5000)\n",
    "    data_word['word_max_'+ feature_word + 'dict_2'] = data_word[feature_word].apply(lambda x : words_to_max(x, dict_2))\n",
    "    data_word['word_std_'+ feature_word + 'dict_2'] = data_word[feature_word].apply(lambda x : words_to_std(x, dict_2))\n",
    "    data_word['word_sum_'+ feature_word + 'dict_2'] = data_word[feature_word].apply(lambda x : words_to_sum(x, dict_2))\n",
    "    data_word['word_last_'+ feature_word + 'dict_2'] = data_word[feature_word].apply(lambda x : words_to_last(x,dict_2))\n",
    "    data_word['word_first_'+ feature_word + 'dict_2'] = data_word[feature_word].apply(lambda x : words_to_first(x, dict_2))\n",
    "    \n",
    "    dict_1 = getdictionary(topK = 10, stop_set = stop_list, label = 1, maxLen = 5000)\n",
    "    data_word['word_max_'+ feature_word + 'dict_1'] = data_word[feature_word].apply(lambda x : words_to_max(x, dict_1))\n",
    "    data_word['word_std_'+ feature_word + 'dict_1'] = data_word[feature_word].apply(lambda x : words_to_std(x, dict_1))\n",
    "    data_word['word_sum_'+ feature_word + 'dict_1'] = data_word[feature_word].apply(lambda x : words_to_sum(x, dict_1))\n",
    "    data_word['word_last_'+ feature_word + 'dict_1'] = data_word[feature_word].apply(lambda x : words_to_last(x,dict_1))\n",
    "    data_word['word_first_'+ feature_word + 'dict_1'] = data_word[feature_word].apply(lambda x : words_to_first(x, dict_1))\n",
    "    \n",
    "    return data_word\n",
    "# 加载 词典集\n",
    "ntusd_pos = read_file_word2set(model_path + 'NTUSD_positive_simplified.txt', encoding = 'utf-8')\n",
    "ntusd_neg = read_file_word2set(model_path + 'NTUSD_negative_simplified.txt', encoding = 'utf-8')\n",
    "\n",
    "# hancks 分词\n",
    "train_word = pd.read_csv(model_path + 'train_word.csv')\n",
    "if testFileExist: \n",
    "    test_word = pd.read_csv(model_path + 'predict_word.csv')\n",
    "    data_word = pd.concat([train_word, test_word])\n",
    "else:\n",
    "    data_word = train_word\n",
    "    \n",
    "stop_list = read_file_word2set(model_path + 'stop_word.txt')\n",
    "# data_word['words'] = data_word['words'].apply(lambda x : clean_str(x, stop_list = stop_list))\n",
    "\n",
    "# 群里siginicant word\n",
    "significant_dictionary =  read_file_word2dict(model_path + 'significance.txt')\n",
    "def contain(word, key):\n",
    "    return 1 if key in word else 0\n",
    "\n",
    "for _, key in enumerate(significant_dictionary.keys()):\n",
    "    data_word['significant_key_{}'.format(_)] = data_word['words'].apply(lambda x : contain(x, key))\n",
    "\n",
    "data_word = data_word_feature(data_word, feature_word = 'words')\n",
    "data = pd.merge(data, data_word.drop(['words'], axis = 1), on = 'Id', how = 'left')\n",
    "\n",
    "# jieba 分词\n",
    "data_jieba = data_word_feature(data_jieba, feature_word = 'words_jieba')\n",
    "data = pd.merge(data, data_jieba.drop(['words_jieba'], axis = 1), on = 'Id', how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Discuss</th>\n",
       "      <th>Score</th>\n",
       "      <th>senti_snownlp</th>\n",
       "      <th>strLen</th>\n",
       "      <th>significant_key_0</th>\n",
       "      <th>significant_key_1</th>\n",
       "      <th>significant_key_2</th>\n",
       "      <th>significant_key_3</th>\n",
       "      <th>significant_key_4</th>\n",
       "      <th>...</th>\n",
       "      <th>word_std_words_jiebadict_2</th>\n",
       "      <th>word_sum_words_jiebadict_2</th>\n",
       "      <th>word_last_words_jiebadict_2</th>\n",
       "      <th>word_first_words_jiebadict_2</th>\n",
       "      <th>word_max_words_jiebadict_1</th>\n",
       "      <th>word_std_words_jiebadict_1</th>\n",
       "      <th>word_sum_words_jiebadict_1</th>\n",
       "      <th>word_last_words_jiebadict_1</th>\n",
       "      <th>word_first_words_jiebadict_1</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>201e8bf2-77a2-3a98-9fcf-4ce03914e712</td>\n",
       "      <td>好大的一个游乐公园，已经去了2次，但感觉还没有玩够似的！会有第三，第四次的</td>\n",
       "      <td>5</td>\n",
       "      <td>0.521543</td>\n",
       "      <td>37</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>428.031077</td>\n",
       "      <td>2314</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1575</td>\n",
       "      <td>328.589048</td>\n",
       "      <td>2083</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f4d51947-eac4-3005-9d3c-2f32d6068a2d</td>\n",
       "      <td>新中国成立也是在这举行，对我们中国人来说有些重要及深刻的意义！</td>\n",
       "      <td>4</td>\n",
       "      <td>0.996441</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>132.925882</td>\n",
       "      <td>1502</td>\n",
       "      <td>317</td>\n",
       "      <td>0</td>\n",
       "      <td>1599</td>\n",
       "      <td>369.789400</td>\n",
       "      <td>2640</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>74aa7ae4-03a4-394c-bee0-5702d3a3082a</td>\n",
       "      <td>庐山瀑布非常有名，也有非常多个瀑布，只是最好看的非三叠泉莫属，推荐一去</td>\n",
       "      <td>4</td>\n",
       "      <td>0.449366</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>589.941052</td>\n",
       "      <td>4434</td>\n",
       "      <td>267</td>\n",
       "      <td>0</td>\n",
       "      <td>434</td>\n",
       "      <td>151.476488</td>\n",
       "      <td>1523</td>\n",
       "      <td>357</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>099661c2-4360-3c49-a2fe-8c783764f7db</td>\n",
       "      <td>个人觉得颐和园是北京最值的一起的地方，不过相比下门票也是最贵的，比起故宫的雄伟与气势磅礴，颐...</td>\n",
       "      <td>5</td>\n",
       "      <td>0.985191</td>\n",
       "      <td>61</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>560.823121</td>\n",
       "      <td>8091</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2389</td>\n",
       "      <td>501.018540</td>\n",
       "      <td>5002</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>97ca672d-e558-3542-ba7b-ee719bba1bab</td>\n",
       "      <td>迪斯尼一日游</td>\n",
       "      <td>5</td>\n",
       "      <td>0.972308</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>552.000000</td>\n",
       "      <td>1104</td>\n",
       "      <td>0</td>\n",
       "      <td>1104</td>\n",
       "      <td>274</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>478</td>\n",
       "      <td>274</td>\n",
       "      <td>204</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 147 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Id  \\\n",
       "0  201e8bf2-77a2-3a98-9fcf-4ce03914e712   \n",
       "1  f4d51947-eac4-3005-9d3c-2f32d6068a2d   \n",
       "2  74aa7ae4-03a4-394c-bee0-5702d3a3082a   \n",
       "3  099661c2-4360-3c49-a2fe-8c783764f7db   \n",
       "4  97ca672d-e558-3542-ba7b-ee719bba1bab   \n",
       "\n",
       "                                             Discuss  Score  senti_snownlp  \\\n",
       "0              好大的一个游乐公园，已经去了2次，但感觉还没有玩够似的！会有第三，第四次的      5       0.521543   \n",
       "1                    新中国成立也是在这举行，对我们中国人来说有些重要及深刻的意义！      4       0.996441   \n",
       "2                庐山瀑布非常有名，也有非常多个瀑布，只是最好看的非三叠泉莫属，推荐一去      4       0.449366   \n",
       "3  个人觉得颐和园是北京最值的一起的地方，不过相比下门票也是最贵的，比起故宫的雄伟与气势磅礴，颐...      5       0.985191   \n",
       "4                                             迪斯尼一日游      5       0.972308   \n",
       "\n",
       "   strLen  significant_key_0  significant_key_1  significant_key_2  \\\n",
       "0      37                  0                  0                  0   \n",
       "1      31                  0                  0                  0   \n",
       "2      35                  0                  0                  0   \n",
       "3      61                  0                  0                  0   \n",
       "4       6                  0                  0                  0   \n",
       "\n",
       "   significant_key_3  significant_key_4    ...      \\\n",
       "0                  0                  0    ...       \n",
       "1                  0                  0    ...       \n",
       "2                  0                  0    ...       \n",
       "3                  0                  0    ...       \n",
       "4                  0                  0    ...       \n",
       "\n",
       "   word_std_words_jiebadict_2  word_sum_words_jiebadict_2  \\\n",
       "0                  428.031077                        2314   \n",
       "1                  132.925882                        1502   \n",
       "2                  589.941052                        4434   \n",
       "3                  560.823121                        8091   \n",
       "4                  552.000000                        1104   \n",
       "\n",
       "   word_last_words_jiebadict_2  word_first_words_jiebadict_2  \\\n",
       "0                            0                             0   \n",
       "1                          317                             0   \n",
       "2                          267                             0   \n",
       "3                            0                             0   \n",
       "4                            0                          1104   \n",
       "\n",
       "   word_max_words_jiebadict_1  word_std_words_jiebadict_1  \\\n",
       "0                        1575                  328.589048   \n",
       "1                        1599                  369.789400   \n",
       "2                         434                  151.476488   \n",
       "3                        2389                  501.018540   \n",
       "4                         274                   35.000000   \n",
       "\n",
       "   word_sum_words_jiebadict_1  word_last_words_jiebadict_1  \\\n",
       "0                        2083                            0   \n",
       "1                        2640                            0   \n",
       "2                        1523                          357   \n",
       "3                        5002                            0   \n",
       "4                         478                          274   \n",
       "\n",
       "   word_first_words_jiebadict_1  sentiment  \n",
       "0                             0          1  \n",
       "1                             0          1  \n",
       "2                             0          0  \n",
       "3                             0          1  \n",
       "4                           204          1  \n",
       "\n",
       "[5 rows x 147 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 情感分析\n",
    "train_senti = pd.read_csv(model_path + \"train_sentiment.csv\")\n",
    "if testFileExist: \n",
    "    test_senti = pd.read_csv(model_path + \"predict_sentiment.csv\")\n",
    "    data_senti = pd.concat([train_senti, test_senti])\n",
    "else:\n",
    "    data_senti = train_senti\n",
    "    \n",
    "data = pd.merge(data, data_senti, on = 'Id', how = 'left')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Discuss</th>\n",
       "      <th>Score</th>\n",
       "      <th>senti_snownlp</th>\n",
       "      <th>strLen</th>\n",
       "      <th>significant_key_0</th>\n",
       "      <th>significant_key_1</th>\n",
       "      <th>significant_key_2</th>\n",
       "      <th>significant_key_3</th>\n",
       "      <th>significant_key_4</th>\n",
       "      <th>...</th>\n",
       "      <th>word_sum_words_jiebadict_2</th>\n",
       "      <th>word_last_words_jiebadict_2</th>\n",
       "      <th>word_first_words_jiebadict_2</th>\n",
       "      <th>word_max_words_jiebadict_1</th>\n",
       "      <th>word_std_words_jiebadict_1</th>\n",
       "      <th>word_sum_words_jiebadict_1</th>\n",
       "      <th>word_last_words_jiebadict_1</th>\n",
       "      <th>word_first_words_jiebadict_1</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>sentiment_self</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>201e8bf2-77a2-3a98-9fcf-4ce03914e712</td>\n",
       "      <td>好大的一个游乐公园，已经去了2次，但感觉还没有玩够似的！会有第三，第四次的</td>\n",
       "      <td>5</td>\n",
       "      <td>0.521543</td>\n",
       "      <td>37</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>2314</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1575</td>\n",
       "      <td>328.589048</td>\n",
       "      <td>2083</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f4d51947-eac4-3005-9d3c-2f32d6068a2d</td>\n",
       "      <td>新中国成立也是在这举行，对我们中国人来说有些重要及深刻的意义！</td>\n",
       "      <td>4</td>\n",
       "      <td>0.996441</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1502</td>\n",
       "      <td>317</td>\n",
       "      <td>0</td>\n",
       "      <td>1599</td>\n",
       "      <td>369.789400</td>\n",
       "      <td>2640</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>74aa7ae4-03a4-394c-bee0-5702d3a3082a</td>\n",
       "      <td>庐山瀑布非常有名，也有非常多个瀑布，只是最好看的非三叠泉莫属，推荐一去</td>\n",
       "      <td>4</td>\n",
       "      <td>0.449366</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>4434</td>\n",
       "      <td>267</td>\n",
       "      <td>0</td>\n",
       "      <td>434</td>\n",
       "      <td>151.476488</td>\n",
       "      <td>1523</td>\n",
       "      <td>357</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>099661c2-4360-3c49-a2fe-8c783764f7db</td>\n",
       "      <td>个人觉得颐和园是北京最值的一起的地方，不过相比下门票也是最贵的，比起故宫的雄伟与气势磅礴，颐...</td>\n",
       "      <td>5</td>\n",
       "      <td>0.985191</td>\n",
       "      <td>61</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>8091</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2389</td>\n",
       "      <td>501.018540</td>\n",
       "      <td>5002</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>97ca672d-e558-3542-ba7b-ee719bba1bab</td>\n",
       "      <td>迪斯尼一日游</td>\n",
       "      <td>5</td>\n",
       "      <td>0.972308</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1104</td>\n",
       "      <td>0</td>\n",
       "      <td>1104</td>\n",
       "      <td>274</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>478</td>\n",
       "      <td>274</td>\n",
       "      <td>204</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 148 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Id  \\\n",
       "0  201e8bf2-77a2-3a98-9fcf-4ce03914e712   \n",
       "1  f4d51947-eac4-3005-9d3c-2f32d6068a2d   \n",
       "2  74aa7ae4-03a4-394c-bee0-5702d3a3082a   \n",
       "3  099661c2-4360-3c49-a2fe-8c783764f7db   \n",
       "4  97ca672d-e558-3542-ba7b-ee719bba1bab   \n",
       "\n",
       "                                             Discuss  Score  senti_snownlp  \\\n",
       "0              好大的一个游乐公园，已经去了2次，但感觉还没有玩够似的！会有第三，第四次的      5       0.521543   \n",
       "1                    新中国成立也是在这举行，对我们中国人来说有些重要及深刻的意义！      4       0.996441   \n",
       "2                庐山瀑布非常有名，也有非常多个瀑布，只是最好看的非三叠泉莫属，推荐一去      4       0.449366   \n",
       "3  个人觉得颐和园是北京最值的一起的地方，不过相比下门票也是最贵的，比起故宫的雄伟与气势磅礴，颐...      5       0.985191   \n",
       "4                                             迪斯尼一日游      5       0.972308   \n",
       "\n",
       "   strLen  significant_key_0  significant_key_1  significant_key_2  \\\n",
       "0      37                  0                  0                  0   \n",
       "1      31                  0                  0                  0   \n",
       "2      35                  0                  0                  0   \n",
       "3      61                  0                  0                  0   \n",
       "4       6                  0                  0                  0   \n",
       "\n",
       "   significant_key_3  significant_key_4       ...        \\\n",
       "0                  0                  0       ...         \n",
       "1                  0                  0       ...         \n",
       "2                  0                  0       ...         \n",
       "3                  0                  0       ...         \n",
       "4                  0                  0       ...         \n",
       "\n",
       "   word_sum_words_jiebadict_2  word_last_words_jiebadict_2  \\\n",
       "0                        2314                            0   \n",
       "1                        1502                          317   \n",
       "2                        4434                          267   \n",
       "3                        8091                            0   \n",
       "4                        1104                            0   \n",
       "\n",
       "   word_first_words_jiebadict_2  word_max_words_jiebadict_1  \\\n",
       "0                             0                        1575   \n",
       "1                             0                        1599   \n",
       "2                             0                         434   \n",
       "3                             0                        2389   \n",
       "4                          1104                         274   \n",
       "\n",
       "   word_std_words_jiebadict_1  word_sum_words_jiebadict_1  \\\n",
       "0                  328.589048                        2083   \n",
       "1                  369.789400                        2640   \n",
       "2                  151.476488                        1523   \n",
       "3                  501.018540                        5002   \n",
       "4                   35.000000                         478   \n",
       "\n",
       "   word_last_words_jiebadict_1  word_first_words_jiebadict_1  sentiment  \\\n",
       "0                            0                             0          1   \n",
       "1                            0                             0          1   \n",
       "2                          357                             0          0   \n",
       "3                            0                             0          1   \n",
       "4                          274                           204          1   \n",
       "\n",
       "   sentiment_self  \n",
       "0               0  \n",
       "1               1  \n",
       "2               1  \n",
       "3               1  \n",
       "4               1  \n",
       "\n",
       "[5 rows x 148 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 情感分析\n",
    "train_senti = pd.read_csv(model_path + \"train_sentiment_self.csv\")\n",
    "if testFileExist:\n",
    "    test_senti = pd.read_csv(model_path + \"predict_sentiment_self.csv\")\n",
    "    data_senti = pd.concat([train_senti, test_senti])\n",
    "else:\n",
    "    data_senti = train_senti\n",
    "    \n",
    "data = pd.merge(data, data_senti, on = 'Id', how = 'left')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "399\n",
      "[['UNK', 63963], ('北京', 4659), ('西湖', 4379), ('中国', 3818), ('黄山', 2976), ('杭州', 2591), ('泰山', 1940), ('故宫', 1758), ('夏天', 1363), ('三清山', 1240), ('天安门', 1098), ('九寨沟', 1062), ('上海', 937), ('张家界', 906), ('北京市', 838), ('西塘', 808), ('都江堰', 779), ('乌镇', 694), ('颐和园', 686), ('鼓浪屿', 651), ('丽江', 632), ('西安', 605), ('南京', 596), ('天池', 590), ('龙门', 537), ('圆明园', 535), ('夫子庙', 523), ('雷峰塔', 514), ('衢州', 506), ('灵隐寺', 484), ('苏州', 480), ('庐山', 451), ('南锣鼓巷', 442), ('石窟', 432), ('青海湖', 428), ('后山', 417), ('绍兴', 410), ('宏村', 406), ('千岛湖', 379), ('宋城', 372), ('天坛', 371), ('名山', 369), ('太湖', 369), ('瘦西湖', 367), ('峨眉山', 367), ('灵山', 367), ('街道', 366), ('大昭寺', 361), ('南天门', 348), ('宁波', 346), ('八达岭', 345), ('青城山', 342), ('中山陵', 341), ('丽江古城', 337), ('布达拉宫', 332), ('秦淮河', 304), ('天门', 304), ('三亚', 300), ('什刹海', 291), ('浙江', 288), ('澳门', 285), ('长白山', 283), ('南山', 283), ('香山', 282), ('漓江', 278), ('广州', 275), ('西海', 274), ('成都', 274), ('东湖', 274), ('武汉', 271), ('月牙泉', 269), ('华山', 268), ('紫禁城', 266), ('厦门', 255), ('鸣沙山', 254), ('一条街', 253), ('南湖', 250), ('黄龙', 248), ('长沙', 246), ('浙江省', 236), ('清朝', 236), ('拉萨', 229), ('崂山', 224), ('众山', 219), ('雍和宫', 217), ('华清池', 216), ('西北', 216), ('四川', 215), ('天门山', 214), ('北海', 213), ('扬州', 212), ('桐庐', 211), ('索桥', 210), ('武夷山', 210), ('昆明湖', 210), ('南海', 204), ('玉龙雪山', 203), ('王府井', 202), ('大理', 202), ('前门', 200)]\n"
     ]
    }
   ],
   "source": [
    "# 地点分析\n",
    "train_address = pd.read_csv(model_path + 'train_address.csv')\n",
    "if testFileExist:\n",
    "    test_address = pd.read_csv(model_path + 'predict_address.csv')\n",
    "    data_address = pd.concat([train_address, test_address])\n",
    "else:\n",
    "    data_address = train_address\n",
    "\n",
    "def max_address(address, dictionary):\n",
    "    address = address[1:-1]\n",
    "    address = address.split(';')\n",
    "    if len(address) == 0: return ''\n",
    "    data = []\n",
    "    for add in address:\n",
    "        if add in dictionary: data.append(dictionary[add])\n",
    "        else: data.append(0)\n",
    "    return np.max(data)\n",
    "\n",
    "def sum_address(address, dictionary):\n",
    "    address = address[1:-1]\n",
    "    address = address.split(';')\n",
    "    if len(address) == 0: return ''\n",
    "    cnt = 0\n",
    "    for add in address:\n",
    "        if add in dictionary: cnt += 1\n",
    "    return cnt\n",
    "\n",
    "def ratio_address(address, dictionary):\n",
    "    address = address[1:-1]\n",
    "    address = address.split(';')\n",
    "    if len(address) == 0: return ''\n",
    "    cnt = 0\n",
    "    for add in address:\n",
    "        if add in dictionary: cnt += 1\n",
    "    return cnt / len(address)\n",
    "\n",
    "\n",
    "# top 100 的旅游景区\n",
    "address_all = get_words(data_address, feature='address_list')\n",
    "word_num, count, dictionary, reverse_dictionary = build_dataset(address_all, vocabulary_size=100)\n",
    "data_address['address_all_max'] = data_address['address_list'].apply(lambda x : max_address(x, dictionary))\n",
    "data_address['address_all_sum'] = data_address['address_list'].apply(lambda x : sum_address(x, dictionary))\n",
    "data_address['address_all_ratio'] = data_address['address_list'].apply(lambda x : ratio_address(x, dictionary))\n",
    "\n",
    "# 差评 旅游景区\n",
    "address = get_neg_words(pd.merge(data_address, data[['Id', 'Score']], on = 'Id', how = 'left'), feature='address_list')\n",
    "word_num_neg, count_neg, dictionary_neg, reverse_dictionary_neg = build_dataset(address, vocabulary_size=100)\n",
    "print(len(address))\n",
    "print(count)\n",
    "data_address['address_neg_all'] = data_address['address_list'].apply(lambda x : max_address(x, dictionary_neg))\n",
    "data_address['address_neg_sum'] = data_address['address_list'].apply(lambda x : sum_address(x, dictionary_neg))\n",
    "data_address['address_neg_ratio'] = data_address['address_list'].apply(lambda x : ratio_address(x, dictionary_neg))\n",
    "\n",
    "del data_address['address_list']\n",
    "data_address.head(100)\n",
    "data = pd.merge(data, data_address,  on = 'Id', how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 载入hankcs分词 情绪窗口\n",
    "senti_window_hankcs = pd.read_csv(model_path + 'data_senti_window_hancks.csv')\n",
    "senti_window_hankcs.head()\n",
    "\n",
    "data = pd.merge(data, senti_window_hankcs, on = 'Id', how = 'left')\n",
    "\n",
    "# 载入结巴分词 情绪窗口\n",
    "senti_window_jieba = pd.read_csv(model_path + 'data_senti_window_jieba.csv')\n",
    "senti_window_jieba.head()\n",
    "\n",
    "data = pd.merge(data, senti_window_jieba, on = 'Id', how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 某个词时候出现过\n",
    "def wordExist(discuss, word = ''):\n",
    "    return 1 if word in discuss else 0\n",
    "\n",
    "data['word_tijian'] = data['Discuss'].apply(lambda x : wordExist(x, '体检'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Discuss</th>\n",
       "      <th>Score</th>\n",
       "      <th>senti_snownlp</th>\n",
       "      <th>strLen</th>\n",
       "      <th>significant_key_0</th>\n",
       "      <th>significant_key_1</th>\n",
       "      <th>significant_key_2</th>\n",
       "      <th>significant_key_3</th>\n",
       "      <th>significant_key_4</th>\n",
       "      <th>...</th>\n",
       "      <th>senti_min_window_5_words_jieba</th>\n",
       "      <th>senti_std_window_5_words_jieba</th>\n",
       "      <th>senti_avg_window_5_words_jieba</th>\n",
       "      <th>senti_median_window_5_words_jieba</th>\n",
       "      <th>senti_max_window_7_words_jieba</th>\n",
       "      <th>senti_min_window_7_words_jieba</th>\n",
       "      <th>senti_std_window_7_words_jieba</th>\n",
       "      <th>senti_avg_window_7_words_jieba</th>\n",
       "      <th>senti_median_window_7_words_jieba</th>\n",
       "      <th>word_tijian</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>201e8bf2-77a2-3a98-9fcf-4ce03914e712</td>\n",
       "      <td>好大的一个游乐公园，已经去了2次，但感觉还没有玩够似的！会有第三，第四次的</td>\n",
       "      <td>5</td>\n",
       "      <td>0.521543</td>\n",
       "      <td>37</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.228059</td>\n",
       "      <td>0.249524</td>\n",
       "      <td>0.502780</td>\n",
       "      <td>0.390102</td>\n",
       "      <td>0.947072</td>\n",
       "      <td>0.124269</td>\n",
       "      <td>0.250820</td>\n",
       "      <td>0.474661</td>\n",
       "      <td>0.389459</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f4d51947-eac4-3005-9d3c-2f32d6068a2d</td>\n",
       "      <td>新中国成立也是在这举行，对我们中国人来说有些重要及深刻的意义！</td>\n",
       "      <td>4</td>\n",
       "      <td>0.996441</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.526233</td>\n",
       "      <td>0.135817</td>\n",
       "      <td>0.765783</td>\n",
       "      <td>0.822042</td>\n",
       "      <td>0.969790</td>\n",
       "      <td>0.526233</td>\n",
       "      <td>0.119520</td>\n",
       "      <td>0.824474</td>\n",
       "      <td>0.873439</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>74aa7ae4-03a4-394c-bee0-5702d3a3082a</td>\n",
       "      <td>庐山瀑布非常有名，也有非常多个瀑布，只是最好看的非三叠泉莫属，推荐一去</td>\n",
       "      <td>4</td>\n",
       "      <td>0.449366</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.058690</td>\n",
       "      <td>0.308193</td>\n",
       "      <td>0.481977</td>\n",
       "      <td>0.419622</td>\n",
       "      <td>0.983864</td>\n",
       "      <td>0.058690</td>\n",
       "      <td>0.329884</td>\n",
       "      <td>0.428539</td>\n",
       "      <td>0.370336</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>099661c2-4360-3c49-a2fe-8c783764f7db</td>\n",
       "      <td>个人觉得颐和园是北京最值的一起的地方，不过相比下门票也是最贵的，比起故宫的雄伟与气势磅礴，颐...</td>\n",
       "      <td>5</td>\n",
       "      <td>0.985191</td>\n",
       "      <td>61</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.212710</td>\n",
       "      <td>0.185634</td>\n",
       "      <td>0.608282</td>\n",
       "      <td>0.581842</td>\n",
       "      <td>0.939794</td>\n",
       "      <td>0.207578</td>\n",
       "      <td>0.191555</td>\n",
       "      <td>0.623142</td>\n",
       "      <td>0.599820</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>97ca672d-e558-3542-ba7b-ee719bba1bab</td>\n",
       "      <td>迪斯尼一日游</td>\n",
       "      <td>5</td>\n",
       "      <td>0.972308</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.972308</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.972308</td>\n",
       "      <td>0.972308</td>\n",
       "      <td>0.972308</td>\n",
       "      <td>0.972308</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.972308</td>\n",
       "      <td>0.972308</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 195 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Id  \\\n",
       "0  201e8bf2-77a2-3a98-9fcf-4ce03914e712   \n",
       "1  f4d51947-eac4-3005-9d3c-2f32d6068a2d   \n",
       "2  74aa7ae4-03a4-394c-bee0-5702d3a3082a   \n",
       "3  099661c2-4360-3c49-a2fe-8c783764f7db   \n",
       "4  97ca672d-e558-3542-ba7b-ee719bba1bab   \n",
       "\n",
       "                                             Discuss  Score  senti_snownlp  \\\n",
       "0              好大的一个游乐公园，已经去了2次，但感觉还没有玩够似的！会有第三，第四次的      5       0.521543   \n",
       "1                    新中国成立也是在这举行，对我们中国人来说有些重要及深刻的意义！      4       0.996441   \n",
       "2                庐山瀑布非常有名，也有非常多个瀑布，只是最好看的非三叠泉莫属，推荐一去      4       0.449366   \n",
       "3  个人觉得颐和园是北京最值的一起的地方，不过相比下门票也是最贵的，比起故宫的雄伟与气势磅礴，颐...      5       0.985191   \n",
       "4                                             迪斯尼一日游      5       0.972308   \n",
       "\n",
       "   strLen  significant_key_0  significant_key_1  significant_key_2  \\\n",
       "0      37                  0                  0                  0   \n",
       "1      31                  0                  0                  0   \n",
       "2      35                  0                  0                  0   \n",
       "3      61                  0                  0                  0   \n",
       "4       6                  0                  0                  0   \n",
       "\n",
       "   significant_key_3  significant_key_4     ...       \\\n",
       "0                  0                  0     ...        \n",
       "1                  0                  0     ...        \n",
       "2                  0                  0     ...        \n",
       "3                  0                  0     ...        \n",
       "4                  0                  0     ...        \n",
       "\n",
       "   senti_min_window_5_words_jieba  senti_std_window_5_words_jieba  \\\n",
       "0                        0.228059                        0.249524   \n",
       "1                        0.526233                        0.135817   \n",
       "2                        0.058690                        0.308193   \n",
       "3                        0.212710                        0.185634   \n",
       "4                        0.972308                        0.000000   \n",
       "\n",
       "   senti_avg_window_5_words_jieba  senti_median_window_5_words_jieba  \\\n",
       "0                        0.502780                           0.390102   \n",
       "1                        0.765783                           0.822042   \n",
       "2                        0.481977                           0.419622   \n",
       "3                        0.608282                           0.581842   \n",
       "4                        0.972308                           0.972308   \n",
       "\n",
       "   senti_max_window_7_words_jieba  senti_min_window_7_words_jieba  \\\n",
       "0                        0.947072                        0.124269   \n",
       "1                        0.969790                        0.526233   \n",
       "2                        0.983864                        0.058690   \n",
       "3                        0.939794                        0.207578   \n",
       "4                        0.972308                        0.972308   \n",
       "\n",
       "   senti_std_window_7_words_jieba  senti_avg_window_7_words_jieba  \\\n",
       "0                        0.250820                        0.474661   \n",
       "1                        0.119520                        0.824474   \n",
       "2                        0.329884                        0.428539   \n",
       "3                        0.191555                        0.623142   \n",
       "4                        0.000000                        0.972308   \n",
       "\n",
       "   senti_median_window_7_words_jieba  word_tijian  \n",
       "0                           0.389459            0  \n",
       "1                           0.873439            0  \n",
       "2                           0.370336            0  \n",
       "3                           0.599820            0  \n",
       "4                           0.972308            0  \n",
       "\n",
       "[5 rows x 195 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bg</th>\n",
       "      <th>mg</th>\n",
       "      <th>nl</th>\n",
       "      <th>nx</th>\n",
       "      <th>qg</th>\n",
       "      <th>ud</th>\n",
       "      <th>uj</th>\n",
       "      <th>uz</th>\n",
       "      <th>ug</th>\n",
       "      <th>ul</th>\n",
       "      <th>...</th>\n",
       "      <th>wf_ratio</th>\n",
       "      <th>wn_ratio</th>\n",
       "      <th>wm_ratio</th>\n",
       "      <th>ws_ratio</th>\n",
       "      <th>wp_ratio</th>\n",
       "      <th>wb_ratio</th>\n",
       "      <th>wh_ratio</th>\n",
       "      <th>end_ratio</th>\n",
       "      <th>begin_ratio</th>\n",
       "      <th>entropy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>130000.0</td>\n",
       "      <td>130000.000000</td>\n",
       "      <td>130000.0</td>\n",
       "      <td>130000.000000</td>\n",
       "      <td>130000.0</td>\n",
       "      <td>130000.0</td>\n",
       "      <td>130000.0</td>\n",
       "      <td>130000.0</td>\n",
       "      <td>130000.0</td>\n",
       "      <td>130000.0</td>\n",
       "      <td>...</td>\n",
       "      <td>130000.0</td>\n",
       "      <td>130000.0</td>\n",
       "      <td>130000.0</td>\n",
       "      <td>130000.0</td>\n",
       "      <td>130000.0</td>\n",
       "      <td>130000.0</td>\n",
       "      <td>130000.0</td>\n",
       "      <td>130000.0</td>\n",
       "      <td>130000.0</td>\n",
       "      <td>130000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000031</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.336431</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.877371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.005547</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.042694</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.812567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.386294</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 297 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             bg             mg        nl             nx        qg        ud  \\\n",
       "count  130000.0  130000.000000  130000.0  130000.000000  130000.0  130000.0   \n",
       "mean        0.0       0.000031       0.0       0.336431       0.0       0.0   \n",
       "std         0.0       0.005547       0.0       2.042694       0.0       0.0   \n",
       "min         0.0       0.000000       0.0       0.000000       0.0       0.0   \n",
       "25%         0.0       0.000000       0.0       0.000000       0.0       0.0   \n",
       "\n",
       "             uj        uz        ug        ul      ...        wf_ratio  \\\n",
       "count  130000.0  130000.0  130000.0  130000.0      ...        130000.0   \n",
       "mean        0.0       0.0       0.0       0.0      ...             0.0   \n",
       "std         0.0       0.0       0.0       0.0      ...             0.0   \n",
       "min         0.0       0.0       0.0       0.0      ...             0.0   \n",
       "25%         0.0       0.0       0.0       0.0      ...             0.0   \n",
       "\n",
       "       wn_ratio  wm_ratio  ws_ratio  wp_ratio  wb_ratio  wh_ratio  end_ratio  \\\n",
       "count  130000.0  130000.0  130000.0  130000.0  130000.0  130000.0   130000.0   \n",
       "mean        0.0       0.0       0.0       0.0       0.0       0.0        0.0   \n",
       "std         0.0       0.0       0.0       0.0       0.0       0.0        0.0   \n",
       "min         0.0       0.0       0.0       0.0       0.0       0.0        0.0   \n",
       "25%         0.0       0.0       0.0       0.0       0.0       0.0        0.0   \n",
       "\n",
       "       begin_ratio        entropy  \n",
       "count     130000.0  130000.000000  \n",
       "mean           0.0       1.877371  \n",
       "std            0.0       0.812567  \n",
       "min            0.0       0.000000  \n",
       "25%            0.0       1.386294  \n",
       "\n",
       "[5 rows x 297 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 属性统计\n",
    "train_property = pd.read_csv(model_path + 'train_property.csv')\n",
    "test_property = pd.read_csv(model_path + 'predict_property.csv')\n",
    "\n",
    "data_property = pd.concat([train_property, test_property])\n",
    "data_property_describe = data_property.describe()\n",
    "data_property_describe.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = data_property_describe.columns\n",
    "filter_columns = []\n",
    "for col in columns:\n",
    "    if data_property_describe.loc['std', col] > 0:\n",
    "        filter_columns.append(col)\n",
    "        \n",
    "filter_columns.append('Id')\n",
    "data = pd.merge(data, data_property[filter_columns], on = 'Id', how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 余弦相似度\n",
    "# data_similarity = pd.read_csv(model_path + 'data_similarity.csv')\n",
    "# data_similarity.head()\n",
    "\n",
    "# data = pd.merge(data, data_similarity, on = 'Id', how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 加载词向量\n",
    "data_vector = pd.read_csv(model_path + 'data_vector.csv')\n",
    "data_vector.describe()\n",
    "\n",
    "data = pd.merge(data, data_vector, on = 'Id', how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 弱模型的特征 无效果\n",
    "# data = pd.merge(data, merge('train_ridge', 'doufu'), on = 'Id', how = 'left')\n",
    "\n",
    "# data = pd.merge(data, merge('train_ridge', '1'), on = 'Id', how = 'left')\n",
    "# data = pd.merge(data, merge('train_ridge', '2'), on = 'Id', how = 'left')\n",
    "# data = pd.merge(data, merge('train_ridge', '3'), on = 'Id', how = 'left')\n",
    "# data = pd.merge(data, merge('train_ridge', '4'), on = 'Id', how = 'left')\n",
    "\n",
    "# data = pd.merge(data, merge('train_lasso', '1'), on = 'Id', how = 'left')\n",
    "# data = pd.merge(data, merge('train_lasso', '2'), on = 'Id', how = 'left')\n",
    "# data = pd.merge(data, merge('train_lasso', '3'), on = 'Id', how = 'left')\n",
    "# data = pd.merge(data, merge('train_lasso', '4'), on = 'Id', how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Discuss</th>\n",
       "      <th>Score</th>\n",
       "      <th>senti_snownlp</th>\n",
       "      <th>strLen</th>\n",
       "      <th>significant_key_0</th>\n",
       "      <th>significant_key_1</th>\n",
       "      <th>significant_key_2</th>\n",
       "      <th>significant_key_3</th>\n",
       "      <th>significant_key_4</th>\n",
       "      <th>...</th>\n",
       "      <th>Vector_190</th>\n",
       "      <th>Vector_191</th>\n",
       "      <th>Vector_192</th>\n",
       "      <th>Vector_193</th>\n",
       "      <th>Vector_194</th>\n",
       "      <th>Vector_195</th>\n",
       "      <th>Vector_196</th>\n",
       "      <th>Vector_197</th>\n",
       "      <th>Vector_198</th>\n",
       "      <th>Vector_199</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>201e8bf2-77a2-3a98-9fcf-4ce03914e712</td>\n",
       "      <td>好大的一个游乐公园，已经去了2次，但感觉还没有玩够似的！会有第三，第四次的</td>\n",
       "      <td>5</td>\n",
       "      <td>0.521543</td>\n",
       "      <td>37</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.004621</td>\n",
       "      <td>0.008280</td>\n",
       "      <td>0.007357</td>\n",
       "      <td>0.020055</td>\n",
       "      <td>-0.036469</td>\n",
       "      <td>0.021086</td>\n",
       "      <td>0.000144</td>\n",
       "      <td>0.017052</td>\n",
       "      <td>-0.021060</td>\n",
       "      <td>0.016037</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f4d51947-eac4-3005-9d3c-2f32d6068a2d</td>\n",
       "      <td>新中国成立也是在这举行，对我们中国人来说有些重要及深刻的意义！</td>\n",
       "      <td>4</td>\n",
       "      <td>0.996441</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.006543</td>\n",
       "      <td>0.063266</td>\n",
       "      <td>-0.056215</td>\n",
       "      <td>-0.013342</td>\n",
       "      <td>-0.056270</td>\n",
       "      <td>-0.006397</td>\n",
       "      <td>0.046615</td>\n",
       "      <td>-0.020320</td>\n",
       "      <td>0.033579</td>\n",
       "      <td>-0.074566</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>74aa7ae4-03a4-394c-bee0-5702d3a3082a</td>\n",
       "      <td>庐山瀑布非常有名，也有非常多个瀑布，只是最好看的非三叠泉莫属，推荐一去</td>\n",
       "      <td>4</td>\n",
       "      <td>0.449366</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.023558</td>\n",
       "      <td>-0.000714</td>\n",
       "      <td>-0.003532</td>\n",
       "      <td>-0.029310</td>\n",
       "      <td>0.049186</td>\n",
       "      <td>0.051639</td>\n",
       "      <td>0.013291</td>\n",
       "      <td>-0.003649</td>\n",
       "      <td>-0.013384</td>\n",
       "      <td>-0.024756</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>099661c2-4360-3c49-a2fe-8c783764f7db</td>\n",
       "      <td>个人觉得颐和园是北京最值的一起的地方，不过相比下门票也是最贵的，比起故宫的雄伟与气势磅礴，颐...</td>\n",
       "      <td>5</td>\n",
       "      <td>0.985191</td>\n",
       "      <td>61</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.004925</td>\n",
       "      <td>-0.004709</td>\n",
       "      <td>-0.020584</td>\n",
       "      <td>-0.025733</td>\n",
       "      <td>-0.033174</td>\n",
       "      <td>0.031373</td>\n",
       "      <td>0.045305</td>\n",
       "      <td>-0.013305</td>\n",
       "      <td>-0.045273</td>\n",
       "      <td>-0.041843</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>97ca672d-e558-3542-ba7b-ee719bba1bab</td>\n",
       "      <td>迪斯尼一日游</td>\n",
       "      <td>5</td>\n",
       "      <td>0.972308</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.002087</td>\n",
       "      <td>-0.010344</td>\n",
       "      <td>-0.076016</td>\n",
       "      <td>0.061931</td>\n",
       "      <td>0.032438</td>\n",
       "      <td>0.066842</td>\n",
       "      <td>-0.062037</td>\n",
       "      <td>0.014033</td>\n",
       "      <td>-0.047095</td>\n",
       "      <td>-0.026730</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 612 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Id  \\\n",
       "0  201e8bf2-77a2-3a98-9fcf-4ce03914e712   \n",
       "1  f4d51947-eac4-3005-9d3c-2f32d6068a2d   \n",
       "2  74aa7ae4-03a4-394c-bee0-5702d3a3082a   \n",
       "3  099661c2-4360-3c49-a2fe-8c783764f7db   \n",
       "4  97ca672d-e558-3542-ba7b-ee719bba1bab   \n",
       "\n",
       "                                             Discuss  Score  senti_snownlp  \\\n",
       "0              好大的一个游乐公园，已经去了2次，但感觉还没有玩够似的！会有第三，第四次的      5       0.521543   \n",
       "1                    新中国成立也是在这举行，对我们中国人来说有些重要及深刻的意义！      4       0.996441   \n",
       "2                庐山瀑布非常有名，也有非常多个瀑布，只是最好看的非三叠泉莫属，推荐一去      4       0.449366   \n",
       "3  个人觉得颐和园是北京最值的一起的地方，不过相比下门票也是最贵的，比起故宫的雄伟与气势磅礴，颐...      5       0.985191   \n",
       "4                                             迪斯尼一日游      5       0.972308   \n",
       "\n",
       "   strLen  significant_key_0  significant_key_1  significant_key_2  \\\n",
       "0      37                  0                  0                  0   \n",
       "1      31                  0                  0                  0   \n",
       "2      35                  0                  0                  0   \n",
       "3      61                  0                  0                  0   \n",
       "4       6                  0                  0                  0   \n",
       "\n",
       "   significant_key_3  significant_key_4     ...      Vector_190  Vector_191  \\\n",
       "0                  0                  0     ...       -0.004621    0.008280   \n",
       "1                  0                  0     ...       -0.006543    0.063266   \n",
       "2                  0                  0     ...       -0.023558   -0.000714   \n",
       "3                  0                  0     ...        0.004925   -0.004709   \n",
       "4                  0                  0     ...       -0.002087   -0.010344   \n",
       "\n",
       "   Vector_192  Vector_193  Vector_194  Vector_195  Vector_196  Vector_197  \\\n",
       "0    0.007357    0.020055   -0.036469    0.021086    0.000144    0.017052   \n",
       "1   -0.056215   -0.013342   -0.056270   -0.006397    0.046615   -0.020320   \n",
       "2   -0.003532   -0.029310    0.049186    0.051639    0.013291   -0.003649   \n",
       "3   -0.020584   -0.025733   -0.033174    0.031373    0.045305   -0.013305   \n",
       "4   -0.076016    0.061931    0.032438    0.066842   -0.062037    0.014033   \n",
       "\n",
       "   Vector_198  Vector_199  \n",
       "0   -0.021060    0.016037  \n",
       "1    0.033579   -0.074566  \n",
       "2   -0.013384   -0.024756  \n",
       "3   -0.045273   -0.041843  \n",
       "4   -0.047095   -0.026730  \n",
       "\n",
       "[5 rows x 612 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 删除 discuss\n",
    "del data['Discuss']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 130000 entries, 0 to 129999\n",
      "Columns: 611 entries, Id to Vector_199\n",
      "dtypes: float64(368), int64(242), object(1)\n",
      "memory usage: 607.0+ MB\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "if not testFileExist:\n",
    "    data.to_csv(model_path + 'data_sub.csv', index = False)\n",
    "    print('persist done...')\n",
    "else:\n",
    "#     data_sub = pd.read_csv(model_path + '/subset12/data_sub.csv')\n",
    "#     data = pd.concat([data, data_sub])\n",
    "    data.to_csv(model_path + 'data.csv', index = False)\n",
    "    print(data.info())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 130000 entries, 0 to 129999\n",
      "Columns: 611 entries, Id to Vector_199\n",
      "dtypes: float64(368), int64(242), object(1)\n",
      "memory usage: 607.0+ MB\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "print(data.info())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\demonsong\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\ipykernel_launcher.py:4: DeprecationWarning: \n",
      ".ix is deprecated. Please use\n",
      ".loc for label based indexing or\n",
      ".iloc for positional indexing\n",
      "\n",
      "See the documentation here:\n",
      "http://pandas.pydata.org/pandas-docs/stable/indexing.html#ix-indexer-is-deprecated\n",
      "  after removing the cwd from sys.path.\n",
      "c:\\users\\demonsong\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\pandas\\core\\indexing.py:194: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self._setitem_with_indexer(indexer, value)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done...\n",
      "done...\n",
      "done...\n",
      "done...\n",
      "done...\n"
     ]
    }
   ],
   "source": [
    "# 划分数据\n",
    "def split_data(data, label):\n",
    "    data_classify_5 = data.copy()\n",
    "    data_classify_5['Score'].ix[(data_classify_5['Score'] != label) & (data_classify_5['Score'] != -1)] = 0\n",
    "    data_classify_5['Score'].ix[data_classify_5['Score'] == label] = 1\n",
    "    data_classify_5.to_csv('../input/multi2binary/' + 'data_{}.csv'.format(label), index = False)\n",
    "    print('done...')\n",
    "\n",
    "for i in range(1, 6):\n",
    "    split_data(data, i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
